From c8cf718719d03c93a76dd669cc46ed97c854e493 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Wed, 4 May 2022 15:18:32 +0200 Subject: [PATCH 001/145] First version - OPT model --- docs/source/en/model_doc/opt.mdx | 113 + src/transformers/__init__.py | 48 + src/transformers/models/__init__.py | 1 + .../models/auto/configuration_auto.py | 3 + src/transformers/models/auto/modeling_auto.py | 8 + .../models/auto/modeling_flax_auto.py | 7 + .../models/auto/modeling_tf_auto.py | 4 + .../models/auto/tokenization_auto.py | 1 + src/transformers/models/opt/__init__.py | 93 + .../models/opt/configuration_opt.py | 398 ++++ ..._original_pytorch_checkpoint_to_pytorch.py | 115 + .../models/opt/modeling_flax_opt.py | 2021 +++++++++++++++++ src/transformers/models/opt/modeling_opt.py | 1818 +++++++++++++++ .../models/opt/modeling_tf_opt.py | 1402 ++++++++++++ .../models/opt/tokenization_opt.py | 417 ++++ .../models/opt/tokenization_opt_fast.py | 303 +++ tests/models/opt/__init__.py | 0 tests/models/opt/test_embeddings_opt.py | 66 + tests/models/opt/test_modeling_flax_opt.py | 567 +++++ tests/models/opt/test_modeling_opt.py | 968 ++++++++ tests/models/opt/test_modeling_tf_opt.py | 477 ++++ tests/models/opt/test_tokenization_opt.py | 185 ++ 22 files changed, 9015 insertions(+) create mode 100644 docs/source/en/model_doc/opt.mdx create mode 100644 src/transformers/models/opt/__init__.py create mode 100644 src/transformers/models/opt/configuration_opt.py create mode 100644 src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py create mode 100644 src/transformers/models/opt/modeling_flax_opt.py create mode 100644 src/transformers/models/opt/modeling_opt.py create mode 100644 src/transformers/models/opt/modeling_tf_opt.py create mode 100644 src/transformers/models/opt/tokenization_opt.py create mode 100644 src/transformers/models/opt/tokenization_opt_fast.py create mode 100644 tests/models/opt/__init__.py create mode 100644 tests/models/opt/test_embeddings_opt.py create mode 100644 tests/models/opt/test_modeling_flax_opt.py create mode 100644 tests/models/opt/test_modeling_opt.py create mode 100644 tests/models/opt/test_modeling_tf_opt.py create mode 100644 tests/models/opt/test_tokenization_opt.py diff --git a/docs/source/en/model_doc/opt.mdx b/docs/source/en/model_doc/opt.mdx new file mode 100644 index 0000000000000..742261ceb02f3 --- /dev/null +++ b/docs/source/en/model_doc/opt.mdx @@ -0,0 +1,113 @@ + + +# OPT + +## Overview + +The OPT model was proposed in []() by . + + +The abstract from the paper is the following: + +** + +Tips: + + + +This model was contributed by [INSERT YOUR HF USERNAME HERE](https://huggingface.co/). +The original code can be found [here](). + + +## OPTConfig + +[[autodoc]] OPTConfig + - all + +## OPTTokenizer + +[[autodoc]] OPTTokenizer + - all + +## OPTTokenizerFast + +[[autodoc]] OPTTokenizerFast + - all + +## OPTModel + +[[autodoc]] OPTModel + - forward + +## OPTForConditionalGeneration + +[[autodoc]] OPTForConditionalGeneration + - forward + +## OPTForSequenceClassification + +[[autodoc]] OPTForSequenceClassification + - forward + +## OPTForQuestionAnswering + +[[autodoc]] OPTForQuestionAnswering + - forward + +## OPTForCausalLM + +[[autodoc]] OPTForCausalLM + - forward + +## TFOPTModel + +[[autodoc]] TFOPTModel + - call + +## TFOPTForConditionalGeneration + +[[autodoc]] TFOPTForConditionalGeneration + - call + +## FlaxOPTModel + +[[autodoc]] FlaxOPTModel + - __call__ + - encode + - decode + +## FlaxOPTForConditionalGeneration + +[[autodoc]] FlaxOPTForConditionalGeneration + - __call__ + - encode + - decode + +## FlaxOPTForSequenceClassification + +[[autodoc]] FlaxOPTForSequenceClassification + - __call__ + - encode + - decode + +## FlaxOPTForQuestionAnswering + +[[autodoc]] FlaxOPTForQuestionAnswering + - __call__ + - encode + - decode + +## FlaxOPTForCausalLM + +[[autodoc]] FlaxOPTForCausalLM + - __call__ \ No newline at end of file diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index ac0f375638a55..e43ba28e9962d 100755 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -131,6 +131,7 @@ "AutoTokenizer", ], "models.bart": ["BartConfig", "BartTokenizer"], + "models.opt": ["OPTConfig", "OPTTokenizer"], "models.barthez": [], "models.bartpho": [], "models.beit": ["BEIT_PRETRAINED_CONFIG_ARCHIVE_MAP", "BeitConfig"], @@ -451,6 +452,7 @@ # Fast tokenizers _import_structure["models.albert"].append("AlbertTokenizerFast") _import_structure["models.bart"].append("BartTokenizerFast") + _import_structure["models.opt"].append("OPTTokenizerFast") _import_structure["models.barthez"].append("BarthezTokenizerFast") _import_structure["models.bert"].append("BertTokenizerFast") _import_structure["models.big_bird"].append("BigBirdTokenizerFast") @@ -731,6 +733,18 @@ "PretrainedBartModel", ] ) + _import_structure["models.opt"].extend( + [ + "OPT_PRETRAINED_MODEL_ARCHIVE_LIST", + "OPTForCausalLM", + "OPTForConditionalGeneration", + "OPTForQuestionAnswering", + "OPTForSequenceClassification", + "OPTModel", + "OPTPretrainedModel", + "PretrainedOPTModel", + ] + ) _import_structure["models.beit"].extend( [ "BEIT_PRETRAINED_MODEL_ARCHIVE_LIST", @@ -1805,6 +1819,7 @@ ] ) _import_structure["models.bart"].extend(["TFBartForConditionalGeneration", "TFBartModel", "TFBartPretrainedModel"]) + _import_structure["models.opt"].extend(["TFOPTForConditionalGeneration", "TFOPTModel", "TFOPTPretrainedModel"]) _import_structure["models.bert"].extend( [ "TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST", @@ -2303,6 +2318,17 @@ "FlaxBartPreTrainedModel", ] ) + _import_structure["models.opt"].extend( + [ + "FlaxOPTDecoderPreTrainedModel", + "FlaxOPTForCausalLM", + "FlaxOPTForConditionalGeneration", + "FlaxOPTForQuestionAnswering", + "FlaxOPTForSequenceClassification", + "FlaxOPTModel", + "FlaxOPTPreTrainedModel", + ] + ) _import_structure["models.beit"].extend( [ @@ -2551,6 +2577,7 @@ AutoTokenizer, ) from .models.bart import BartConfig, BartTokenizer + from .models.opt import OPTConfig, OPTTokenizer from .models.beit import BEIT_PRETRAINED_CONFIG_ARCHIVE_MAP, BeitConfig from .models.bert import ( BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, @@ -2840,6 +2867,7 @@ if is_tokenizers_available(): from .models.albert import AlbertTokenizerFast from .models.bart import BartTokenizerFast + from .models.opt import OPTTokenizerFast from .models.barthez import BarthezTokenizerFast from .models.bert import BertTokenizerFast from .models.big_bird import BigBirdTokenizerFast @@ -3069,6 +3097,16 @@ BartPretrainedModel, PretrainedBartModel, ) + from .models.opt import ( + OPT_PRETRAINED_MODEL_ARCHIVE_LIST, + OPTForCausalLM, + OPTForConditionalGeneration, + OPTForQuestionAnswering, + OPTForSequenceClassification, + OPTModel, + OPTPretrainedModel, + PretrainedOPTModel, + ) from .models.beit import ( BEIT_PRETRAINED_MODEL_ARCHIVE_LIST, BeitForImageClassification, @@ -3968,6 +4006,7 @@ TFAutoModelWithLMHead, ) from .models.bart import TFBartForConditionalGeneration, TFBartModel, TFBartPretrainedModel + from .models.opt import TFOPTForConditionalGeneration, TFOPTModel, TFOPTPretrainedModel from .models.bert import ( TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST, TFBertEmbeddings, @@ -4360,6 +4399,15 @@ FlaxBartModel, FlaxBartPreTrainedModel, ) + from .models.opt import ( + FlaxOPTDecoderPreTrainedModel, + FlaxOPTForCausalLM, + FlaxOPTForConditionalGeneration, + FlaxOPTForQuestionAnswering, + FlaxOPTForSequenceClassification, + FlaxOPTModel, + FlaxOPTPreTrainedModel, + ) from .models.beit import ( FlaxBeitForImageClassification, FlaxBeitForMaskedImageModeling, diff --git a/src/transformers/models/__init__.py b/src/transformers/models/__init__.py index 8697e4333d67d..93922f7d901fa 100644 --- a/src/transformers/models/__init__.py +++ b/src/transformers/models/__init__.py @@ -20,6 +20,7 @@ albert, auto, bart, + opt, barthez, bartpho, beit, diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index 0eaa838d47024..214db9a73abf5 100644 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -97,6 +97,7 @@ ("megatron-bert", "MegatronBertConfig"), ("mpnet", "MPNetConfig"), ("bart", "BartConfig"), + ("opt", "OPTConfig"), ("blenderbot", "BlenderbotConfig"), ("reformer", "ReformerConfig"), ("longformer", "LongformerConfig"), @@ -188,6 +189,7 @@ ("blenderbot-small", "BLENDERBOT_SMALL_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("bert", "BERT_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("bart", "BART_PRETRAINED_CONFIG_ARCHIVE_MAP"), + ("opt", "OPT_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("blenderbot", "BLENDERBOT_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("mbart", "MBART_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("openai-gpt", "OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP"), @@ -298,6 +300,7 @@ ("mbart", "mBART"), ("megatron-bert", "MegatronBert"), ("bart", "BART"), + ("opt", "OPT"), ("reformer", "Reformer"), ("longformer", "Longformer"), ("roberta", "RoBERTa"), diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index 13f1070c6cc6a..cfe1cb1b027b9 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -92,6 +92,7 @@ ("xlm-roberta-xl", "XLMRobertaXLModel"), ("xlm-roberta", "XLMRobertaModel"), ("bart", "BartModel"), + ("opt", "OPTModel"), ("longformer", "LongformerModel"), ("roberta", "RobertaModel"), ("data2vec-text", "Data2VecTextModel"), @@ -143,6 +144,7 @@ ("xlm-roberta-xl", "XLMRobertaXLForMaskedLM"), ("xlm-roberta", "XLMRobertaForMaskedLM"), ("bart", "BartForConditionalGeneration"), + ("opt", "OPTForConditionalGeneration"), ("fsmt", "FSMTForConditionalGeneration"), ("longformer", "LongformerForMaskedLM"), ("roberta", "RobertaForMaskedLM"), @@ -203,6 +205,7 @@ ("marian", "MarianMTModel"), ("fsmt", "FSMTForConditionalGeneration"), ("bart", "BartForConditionalGeneration"), + ("opt", "OPTForConditionalGeneration"), ("longformer", "LongformerForMaskedLM"), ("roberta", "RobertaForMaskedLM"), ("data2vec-text", "Data2VecTextForMaskedLM"), @@ -259,6 +262,7 @@ ("xlm-prophetnet", "XLMProphetNetForCausalLM"), ("prophetnet", "ProphetNetForCausalLM"), ("bart", "BartForCausalLM"), + ("opt", "OPTForCausalLM"), ("mbart", "MBartForCausalLM"), ("pegasus", "PegasusForCausalLM"), ("marian", "MarianForCausalLM"), @@ -360,6 +364,7 @@ ("distilbert", "DistilBertForMaskedLM"), ("albert", "AlbertForMaskedLM"), ("bart", "BartForConditionalGeneration"), + ("opt", "OPTForConditionalGeneration"), ("mbart", "MBartForConditionalGeneration"), ("camembert", "CamembertForMaskedLM"), ("xlm-roberta-xl", "XLMRobertaXLForMaskedLM"), @@ -408,6 +413,7 @@ ("mbart", "MBartForConditionalGeneration"), ("blenderbot", "BlenderbotForConditionalGeneration"), ("bart", "BartForConditionalGeneration"), + ("opt", "OPTForConditionalGeneration"), ("fsmt", "FSMTForConditionalGeneration"), ("encoder-decoder", "EncoderDecoderModel"), ("xlm-prophetnet", "XLMProphetNetForConditionalGeneration"), @@ -448,6 +454,7 @@ ("xlm-roberta", "XLMRobertaForSequenceClassification"), ("mbart", "MBartForSequenceClassification"), ("bart", "BartForSequenceClassification"), + ("opt", "OPTForSequenceClassification"), ("longformer", "LongformerForSequenceClassification"), ("roberta", "RobertaForSequenceClassification"), ("data2vec-text", "Data2VecTextForSequenceClassification"), @@ -495,6 +502,7 @@ ("albert", "AlbertForQuestionAnswering"), ("camembert", "CamembertForQuestionAnswering"), ("bart", "BartForQuestionAnswering"), + ("opt", "OPTForQuestionAnswering"), ("mbart", "MBartForQuestionAnswering"), ("longformer", "LongformerForQuestionAnswering"), ("xlm-roberta-xl", "XLMRobertaXLForQuestionAnswering"), diff --git a/src/transformers/models/auto/modeling_flax_auto.py b/src/transformers/models/auto/modeling_flax_auto.py index 78803178bec0c..1c6972eea5b1a 100644 --- a/src/transformers/models/auto/modeling_flax_auto.py +++ b/src/transformers/models/auto/modeling_flax_auto.py @@ -40,6 +40,7 @@ ("beit", "FlaxBeitModel"), ("big_bird", "FlaxBigBirdModel"), ("bart", "FlaxBartModel"), + ("opt", "FlaxOPTModel"), ("gpt2", "FlaxGPT2Model"), ("gpt_neo", "FlaxGPTNeoModel"), ("gptj", "FlaxGPTJModel"), @@ -65,6 +66,7 @@ ("bert", "FlaxBertForPreTraining"), ("big_bird", "FlaxBigBirdForPreTraining"), ("bart", "FlaxBartForConditionalGeneration"), + ("opt", "FlaxOPTForConditionalGeneration"), ("electra", "FlaxElectraForPreTraining"), ("mbart", "FlaxMBartForConditionalGeneration"), ("t5", "FlaxT5ForConditionalGeneration"), @@ -84,6 +86,7 @@ ("bert", "FlaxBertForMaskedLM"), ("big_bird", "FlaxBigBirdForMaskedLM"), ("bart", "FlaxBartForConditionalGeneration"), + ("opt", "FlaxOPTForConditionalGeneration"), ("electra", "FlaxElectraForMaskedLM"), ("mbart", "FlaxMBartForConditionalGeneration"), ("roformer", "FlaxRoFormerForMaskedLM"), @@ -96,6 +99,7 @@ ("blenderbot-small", "FlaxBlenderbotSmallForConditionalGeneration"), ("pegasus", "FlaxPegasusForConditionalGeneration"), ("bart", "FlaxBartForConditionalGeneration"), + ("opt", "FlaxOPTForConditionalGeneration"), ("mbart", "FlaxMBartForConditionalGeneration"), ("t5", "FlaxT5ForConditionalGeneration"), ("mt5", "FlaxMT5ForConditionalGeneration"), @@ -127,6 +131,7 @@ ("gptj", "FlaxGPTJForCausalLM"), ("xglm", "FlaxXGLMForCausalLM"), ("bart", "FlaxBartForCausalLM"), + ("opt", "FlaxOPTForCausalLM"), ("bert", "FlaxBertForCausalLM"), ("roberta", "FlaxRobertaForCausalLM"), ("big_bird", "FlaxBigBirdForCausalLM"), @@ -144,6 +149,7 @@ ("bert", "FlaxBertForSequenceClassification"), ("big_bird", "FlaxBigBirdForSequenceClassification"), ("bart", "FlaxBartForSequenceClassification"), + ("opt", "FlaxOPTForSequenceClassification"), ("electra", "FlaxElectraForSequenceClassification"), ("mbart", "FlaxMBartForSequenceClassification"), ("roformer", "FlaxRoFormerForSequenceClassification"), @@ -160,6 +166,7 @@ ("bert", "FlaxBertForQuestionAnswering"), ("big_bird", "FlaxBigBirdForQuestionAnswering"), ("bart", "FlaxBartForQuestionAnswering"), + ("opt", "FlaxOPTForQuestionAnswering"), ("electra", "FlaxElectraForQuestionAnswering"), ("mbart", "FlaxMBartForQuestionAnswering"), ("roformer", "FlaxRoFormerForQuestionAnswering"), diff --git a/src/transformers/models/auto/modeling_tf_auto.py b/src/transformers/models/auto/modeling_tf_auto.py index a806086cea1ed..d5e74dd3877a5 100644 --- a/src/transformers/models/auto/modeling_tf_auto.py +++ b/src/transformers/models/auto/modeling_tf_auto.py @@ -44,6 +44,7 @@ ("distilbert", "TFDistilBertModel"), ("albert", "TFAlbertModel"), ("bart", "TFBartModel"), + ("opt", "TFOPTModel"), ("camembert", "TFCamembertModel"), ("xlm-roberta", "TFXLMRobertaModel"), ("longformer", "TFLongformerModel"), @@ -84,6 +85,7 @@ ("distilbert", "TFDistilBertForMaskedLM"), ("albert", "TFAlbertForPreTraining"), ("bart", "TFBartForConditionalGeneration"), + ("opt", "TFOPTForConditionalGeneration"), ("camembert", "TFCamembertForMaskedLM"), ("xlm-roberta", "TFXLMRobertaForMaskedLM"), ("roberta", "TFRobertaForMaskedLM"), @@ -118,6 +120,7 @@ ("albert", "TFAlbertForMaskedLM"), ("marian", "TFMarianMTModel"), ("bart", "TFBartForConditionalGeneration"), + ("opt", "TFOPTForConditionalGeneration"), ("camembert", "TFCamembertForMaskedLM"), ("xlm-roberta", "TFXLMRobertaForMaskedLM"), ("longformer", "TFLongformerForMaskedLM"), @@ -210,6 +213,7 @@ ("blenderbot", "TFBlenderbotForConditionalGeneration"), ("blenderbot-small", "TFBlenderbotSmallForConditionalGeneration"), ("bart", "TFBartForConditionalGeneration"), + ("opt", "TFOPTForConditionalGeneration"), ("encoder-decoder", "TFEncoderDecoderModel"), ] ) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 0c0059c7c6575..7d688ba13ff70 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -107,6 +107,7 @@ ("blenderbot", ("BlenderbotTokenizer", "BlenderbotTokenizerFast")), ("tapex", ("TapexTokenizer", None)), ("bart", ("BartTokenizer", "BartTokenizerFast")), + ("opt", ("OPTTokenizer", "OPTTokenizerFast")), ("longformer", ("LongformerTokenizer", "LongformerTokenizerFast" if is_tokenizers_available() else None)), ("roberta", ("RobertaTokenizer", "RobertaTokenizerFast" if is_tokenizers_available() else None)), ( diff --git a/src/transformers/models/opt/__init__.py b/src/transformers/models/opt/__init__.py new file mode 100644 index 0000000000000..641764a29b341 --- /dev/null +++ b/src/transformers/models/opt/__init__.py @@ -0,0 +1,93 @@ +# flake8: noqa +# There's no way to ignore "F401 '...' imported but unused" warnings in this +# module, but to preserve other warnings. So, don't check this module at all. + +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +from ...utils import _LazyModule, is_flax_available, is_tf_available, is_tokenizers_available, is_torch_available + + +_import_structure = { + "configuration_opt": ["OPT_PRETRAINED_CONFIG_ARCHIVE_MAP", "OPTConfig", "OPTOnnxConfig"], + "tokenization_opt": ["OPTTokenizer"], +} + +if is_tokenizers_available(): + _import_structure["tokenization_opt_fast"] = ["OPTTokenizerFast"] + +if is_torch_available(): + _import_structure["modeling_opt"] = [ + "OPT_PRETRAINED_MODEL_ARCHIVE_LIST", + "OPTForCausalLM", + "OPTForConditionalGeneration", + "OPTForQuestionAnswering", + "OPTForSequenceClassification", + "OPTModel", + "OPTPretrainedModel", + "PretrainedOPTModel", + ] + +if is_tf_available(): + _import_structure["modeling_tf_opt"] = ["TFOPTForConditionalGeneration", "TFOPTModel", "TFOPTPretrainedModel"] + +if is_flax_available(): + _import_structure["modeling_flax_opt"] = [ + "FlaxOPTDecoderPreTrainedModel", + "FlaxOPTForCausalLM", + "FlaxOPTForConditionalGeneration", + "FlaxOPTForQuestionAnswering", + "FlaxOPTForSequenceClassification", + "FlaxOPTModel", + "FlaxOPTPreTrainedModel", + ] + +if TYPE_CHECKING: + from .configuration_opt import OPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OPTConfig, OPTOnnxConfig + from .tokenization_opt import OPTTokenizer + + if is_tokenizers_available(): + from .tokenization_opt_fast import OPTTokenizerFast + + if is_torch_available(): + from .modeling_opt import ( + OPT_PRETRAINED_MODEL_ARCHIVE_LIST, + OPTForCausalLM, + OPTForConditionalGeneration, + OPTForQuestionAnswering, + OPTForSequenceClassification, + OPTModel, + OPTPretrainedModel, + PretrainedOPTModel, + ) + + if is_tf_available(): + from .modeling_tf_opt import TFOPTForConditionalGeneration, TFOPTModel, TFOPTPretrainedModel + + if is_flax_available(): + from .modeling_flax_opt import ( + FlaxOPTDecoderPreTrainedModel, + FlaxOPTForCausalLM, + FlaxOPTForConditionalGeneration, + FlaxOPTForQuestionAnswering, + FlaxOPTForSequenceClassification, + FlaxOPTModel, + FlaxOPTPreTrainedModel, + ) + +else: + import sys + + sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__) diff --git a/src/transformers/models/opt/configuration_opt.py b/src/transformers/models/opt/configuration_opt.py new file mode 100644 index 0000000000000..7a5ad805dc6a5 --- /dev/null +++ b/src/transformers/models/opt/configuration_opt.py @@ -0,0 +1,398 @@ +# coding=utf-8 +# Copyright 2021 The Fairseq Authors and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" OPT model configuration""" +import warnings +from collections import OrderedDict +from typing import Any, Mapping, Optional + +from ... import PreTrainedTokenizer +from ...configuration_utils import PretrainedConfig +from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeq2SeqConfigWithPast +from ...onnx.utils import compute_effective_axis_dimension +from ...utils import TensorType, is_torch_available, logging + + +logger = logging.get_logger(__name__) + +OPT_PRETRAINED_CONFIG_ARCHIVE_MAP = { + "": "https://huggingface.co//resolve/main/config.json", +} + + + +class OPTConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`OPTModel`]. It is used to instantiate a OPT + model according to the specified arguments, defining the model architecture. Instantiating a configuration with the + defaults will yield a similar configuration to that of the OPT + [facebook/opt-large](https://huggingface.co/facebook/opt-large) architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 50265): + Vocabulary size of the OPT model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`OPTModel`] or [`TFOPTModel`]. + d_model (`int`, *optional*, defaults to 1024): + Dimensionality of the layers and the pooler layer. + encoder_layers (`int`, *optional*, defaults to 12): + Number of encoder layers. + decoder_layers (`int`, *optional*, defaults to 12): + Number of decoder layers. + encoder_attention_heads (`int`, *optional*, defaults to 16): + Number of attention heads for each attention layer in the Transformer encoder. + decoder_attention_heads (`int`, *optional*, defaults to 16): + Number of attention heads for each attention layer in the Transformer decoder. + decoder_ffn_dim (`int`, *optional*, defaults to 4096): + Dimensionality of the "intermediate" (often named feed-forward) layer in decoder. + encoder_ffn_dim (`int`, *optional*, defaults to 4096): + Dimensionality of the "intermediate" (often named feed-forward) layer in decoder. + activation_function (`str` or `function`, *optional*, defaults to `"gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"silu"` and `"gelu_new"` are supported. + dropout (`float`, *optional*, defaults to 0.1): + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + activation_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for activations inside the fully connected layer. + classifier_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for classifier. + max_position_embeddings (`int`, *optional*, defaults to 1024): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + init_std (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + encoder_layerdrop: (`float`, *optional*, defaults to 0.0): + The LayerDrop probability for the encoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556) + for more details. + decoder_layerdrop: (`float`, *optional*, defaults to 0.0): + The LayerDrop probability for the decoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556) + for more details. + scale_embedding (`bool`, *optional*, defaults to `False`): + Scale embeddings by diving by sqrt(d_model). + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). + num_labels: (`int`, *optional*, defaults to 3): + The number of labels to use in [`OPTForSequenceClassification`]. + forced_eos_token_id (`int`, *optional*, defaults to 2): + The id of the token to force as the last generated token when `max_length` is reached. Usually set to + `eos_token_id`. + + Example: + + ```python + >>> from transformers import OPTModel, OPTConfig + + >>> # Initializing a OPT facebook/opt-large style configuration + >>> configuration = OPTConfig() + + >>> # Initializing a model from the facebook/opt-large style configuration + >>> model = OPTModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + model_type = "opt" + keys_to_ignore_at_inference = ["past_key_values"] + attribute_map = {"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"} + + def __init__( + self, + vocab_size=50272, # TODO check the real value + max_position_embeddings=2048, + decoder_layers=24, + decoder_ffn_dim=4096, + decoder_attention_heads=16, + decoder_layerdrop=0.0, + activation_function="gelu", + d_model=1024, + embed_dim=512, + dropout=0.1, + attention_dropout=0.0, + activation_dropout=0.0, + init_std=0.02, + classifier_dropout=0.0, + scale_embedding=False, + share_input_output_embed=True, + use_cache=True, + num_labels=3, + pad_token_id=1, + bos_token_id=0, + eos_token_id=2, + is_encoder_decoder=True, + decoder_start_token_id=2, + forced_eos_token_id=2, + **kwargs + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.embed_dim = embed_dim + self.share_input_output_embed = share_input_output_embed + self.d_model = d_model + self.decoder_ffn_dim = decoder_ffn_dim + self.decoder_layers = decoder_layers + self.decoder_attention_heads = decoder_attention_heads + self.dropout = dropout + self.attention_dropout = attention_dropout + self.activation_dropout = activation_dropout + self.activation_function = activation_function + self.init_std = init_std + self.decoder_layerdrop = decoder_layerdrop + self.classifier_dropout = classifier_dropout + self.use_cache = use_cache + self.scale_embedding = scale_embedding # scale factor will be sqrt(d_model) if True + + super().__init__( + num_labels=num_labels, + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + is_encoder_decoder=is_encoder_decoder, + decoder_start_token_id=decoder_start_token_id, + forced_eos_token_id=forced_eos_token_id, + **kwargs, + ) + + # ensure backward compatibility for OPT CNN models + if self.forced_bos_token_id is None and kwargs.get("force_bos_token_to_be_generated", False): + self.forced_bos_token_id = self.bos_token_id + warnings.warn( + f"Please make sure the config includes `forced_bos_token_id={self.bos_token_id}` in future versions. " + "The config can simply be saved and uploaded again to be fixed." + ) + + +class OPTOnnxConfig(OnnxSeq2SeqConfigWithPast): + @property + def inputs(self) -> Mapping[str, Mapping[int, str]]: + if self.task in ["default", "seq2seq-lm"]: + common_inputs = OrderedDict( + [ + ("input_ids", {0: "batch", 1: "encoder_sequence"}), + ("attention_mask", {0: "batch", 1: "encoder_sequence"}), + ] + ) + + if self.use_past: + common_inputs["decoder_input_ids"] = {0: "batch"} + common_inputs["decoder_attention_mask"] = {0: "batch", 1: "past_decoder_sequence + sequence"} + else: + common_inputs["decoder_input_ids"] = {0: "batch", 1: "decoder_sequence"} + common_inputs["decoder_attention_mask"] = {0: "batch", 1: "decoder_sequence"} + + if self.use_past: + self.fill_with_past_key_values_(common_inputs, direction="inputs") + elif self.task == "causal-lm": + # TODO: figure this case out. + common_inputs = OrderedDict( + [ + ("input_ids", {0: "batch", 1: "encoder_sequence"}), + ("attention_mask", {0: "batch", 1: "encoder_sequence"}), + ] + ) + if self.use_past: + num_encoder_layers, _ = self.num_layers + for i in range(num_encoder_layers): + common_inputs[f"past_key_values.{i}.key"] = {0: "batch", 2: "past_sequence + sequence"} + common_inputs[f"past_key_values.{i}.value"] = {0: "batch", 2: "past_sequence + sequence"} + else: + common_inputs = OrderedDict( + [ + ("input_ids", {0: "batch", 1: "encoder_sequence"}), + ("attention_mask", {0: "batch", 1: "encoder_sequence"}), + ("decoder_input_ids", {0: "batch", 1: "decoder_sequence"}), + ("decoder_attention_mask", {0: "batch", 1: "decoder_sequence"}), + ] + ) + + return common_inputs + + @property + def outputs(self) -> Mapping[str, Mapping[int, str]]: + if self.task in ["default", "seq2seq-lm"]: + common_outputs = super().outputs + else: + common_outputs = super(OnnxConfigWithPast, self).outputs + if self.use_past: + num_encoder_layers, _ = self.num_layers + for i in range(num_encoder_layers): + common_outputs[f"present.{i}.key"] = {0: "batch", 2: "past_sequence + sequence"} + common_outputs[f"present.{i}.value"] = {0: "batch", 2: "past_sequence + sequence"} + return common_outputs + + def _generate_dummy_inputs_for_default_and_seq2seq_lm( + self, + tokenizer: PreTrainedTokenizer, + batch_size: int = -1, + seq_length: int = -1, + is_pair: bool = False, + framework: Optional[TensorType] = None, + ) -> Mapping[str, Any]: + encoder_inputs = self._generate_dummy_inputs_for_sequence_classification_and_question_answering( + tokenizer, batch_size, seq_length, is_pair, framework + ) + + # Generate decoder inputs + decoder_seq_length = seq_length if not self.use_past else 1 + decoder_inputs = self._generate_dummy_inputs_for_sequence_classification_and_question_answering( + tokenizer, batch_size, decoder_seq_length, is_pair, framework + ) + decoder_inputs = {f"decoder_{name}": tensor for name, tensor in decoder_inputs.items()} + common_inputs = dict(**encoder_inputs, **decoder_inputs) + + if self.use_past: + if not is_torch_available(): + raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed.") + else: + import torch + batch, encoder_seq_length = common_inputs["input_ids"].shape + decoder_seq_length = common_inputs["decoder_input_ids"].shape[1] + num_encoder_attention_heads, num_decoder_attention_heads = self.num_attention_heads + encoder_shape = ( + batch, + num_encoder_attention_heads, + encoder_seq_length, + self._config.hidden_size // num_encoder_attention_heads, + ) + decoder_past_length = decoder_seq_length + 3 + decoder_shape = ( + batch, + num_decoder_attention_heads, + decoder_past_length, + self._config.hidden_size // num_decoder_attention_heads, + ) + + common_inputs["decoder_attention_mask"] = torch.cat( + [common_inputs["decoder_attention_mask"], torch.ones(batch, decoder_past_length)], dim=1 + ) + + common_inputs["past_key_values"] = [] + # If the number of encoder and decoder layers are present in the model configuration, both are considered + num_encoder_layers, num_decoder_layers = self.num_layers + min_num_layers = min(num_encoder_layers, num_decoder_layers) + max_num_layers = max(num_encoder_layers, num_decoder_layers) - min_num_layers + remaining_side_name = "encoder" if num_encoder_layers > num_decoder_layers else "decoder" + + for _ in range(min_num_layers): + common_inputs["past_key_values"].append( + ( + torch.zeros(decoder_shape), + torch.zeros(decoder_shape), + torch.zeros(encoder_shape), + torch.zeros(encoder_shape), + ) + ) + # TODO: test this. + shape = encoder_shape if remaining_side_name == "encoder" else decoder_shape + for _ in range(min_num_layers, max_num_layers): + common_inputs["past_key_values"].append((torch.zeros(shape), torch.zeros(shape))) + return common_inputs + + def _generate_dummy_inputs_for_causal_lm( + self, + tokenizer: PreTrainedTokenizer, + batch_size: int = -1, + seq_length: int = -1, + is_pair: bool = False, + framework: Optional[TensorType] = None, + ) -> Mapping[str, Any]: + common_inputs = self._generate_dummy_inputs_for_sequence_classification_and_question_answering( + tokenizer, batch_size, seq_length, is_pair, framework + ) + + if self.use_past: + if not is_torch_available(): + raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed.") + else: + import torch + batch, seqlen = common_inputs["input_ids"].shape + # Not using the same length for past_key_values + past_key_values_length = seqlen + 2 + num_encoder_layers, _ = self.num_layers + num_encoder_attention_heads, _ = self.num_attention_heads + past_shape = ( + batch, + num_encoder_attention_heads, + past_key_values_length, + self._config.hidden_size // num_encoder_attention_heads, + ) + + common_inputs["attention_mask"] = torch.cat( + [common_inputs["attention_mask"], torch.ones(batch, past_key_values_length)], dim=1 + ) + common_inputs["past_key_values"] = [ + (torch.zeros(past_shape), torch.zeros(past_shape)) for _ in range(num_encoder_layers) + ] + return common_inputs + + def _generate_dummy_inputs_for_sequence_classification_and_question_answering( + self, + tokenizer: PreTrainedTokenizer, + batch_size: int = -1, + seq_length: int = -1, + is_pair: bool = False, + framework: Optional[TensorType] = None, + ) -> Mapping[str, Any]: + # Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity. + # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX + batch_size = compute_effective_axis_dimension( + batch_size, fixed_dimension=OnnxConfig.default_fixed_batch, num_token_to_add=0 + ) + + # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX + token_to_add = tokenizer.num_special_tokens_to_add(is_pair) + seq_length = compute_effective_axis_dimension( + seq_length, fixed_dimension=OnnxConfig.default_fixed_sequence, num_token_to_add=token_to_add + ) + + # Generate dummy inputs according to compute batch and sequence + dummy_input = [" ".join([tokenizer.unk_token]) * seq_length] * batch_size + common_inputs = dict(tokenizer(dummy_input, return_tensors=framework)) + return common_inputs + + def generate_dummy_inputs( + self, + tokenizer: PreTrainedTokenizer, + batch_size: int = -1, + seq_length: int = -1, + is_pair: bool = False, + framework: Optional[TensorType] = None, + ) -> Mapping[str, Any]: + if self.task in ["default", "seq2seq-lm"]: + common_inputs = self._generate_dummy_inputs_for_default_and_seq2seq_lm( + tokenizer, batch_size=batch_size, seq_length=seq_length, is_pair=is_pair, framework=framework + ) + + elif self.task == "causal-lm": + common_inputs = self._generate_dummy_inputs_for_causal_lm( + tokenizer, batch_size=batch_size, seq_length=seq_length, is_pair=is_pair, framework=framework + ) + else: + common_inputs = self._generate_dummy_inputs_for_sequence_classification_and_question_answering( + tokenizer, batch_size=batch_size, seq_length=seq_length, is_pair=is_pair, framework=framework + ) + + return common_inputs + + def _flatten_past_key_values_(self, flattened_output, name, idx, t): + if self.task in ["default", "seq2seq-lm"]: + flattened_output = super()._flatten_past_key_values_(flattened_output, name, idx, t) + else: + flattened_output = super(OnnxSeq2SeqConfigWithPast, self)._flatten_past_key_values_( + flattened_output, name, idx, t + ) diff --git a/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py new file mode 100644 index 0000000000000..088592f2c9ea0 --- /dev/null +++ b/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py @@ -0,0 +1,115 @@ +# coding=utf-8 +# Copyright 2020 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Convert OPT checkpoint.""" + + +import argparse +import os +from pathlib import Path + +# import fairseq +import torch +from packaging import version +from torch import nn + +from transformers import ( + OPTConfig, + OPTForConditionalGeneration, + OPTForSequenceClassification, + OPTModel, + OPTTokenizer, +) +from transformers.utils import logging + + +FAIRSEQ_MODELS = ["opt.large", "opt.large.mnli", "opt.large.cnn", "opt_xsum/model.pt"] +extra_arch = {"opt.large": OPTModel, "opt.large.mnli": OPTForSequenceClassification} +# if version.parse(fairseq.__version__) < version.parse("0.9.0"): +# raise Exception("requires fairseq >= 0.9.0") + + +logging.set_verbosity_info() +logger = logging.get_logger(__name__) + +SAMPLE_TEXT = " Hello world! cécé herlolip" + +mnli_rename_keys = [ + ("model.classification_heads.mnli.dense.weight", "classification_head.dense.weight"), + ("model.classification_heads.mnli.dense.bias", "classification_head.dense.bias"), + ("model.classification_heads.mnli.out_proj.weight", "classification_head.out_proj.weight"), + ("model.classification_heads.mnli.out_proj.bias", "classification_head.out_proj.bias"), +] + + +def remove_ignore_keys_(state_dict): + ignore_keys = [ + "encoder.version", + "decoder.version", + "model.encoder.version", + "model.decoder.version", + "_float_tensor", + ] + for k in ignore_keys: + state_dict.pop(k, None) + + +def rename_key(dct, old, new): + val = dct.pop(old) + dct[new] = val + + +def load_xsum_checkpoint(checkpoint_path): + """Checkpoint path should end in model.pt""" + sd = torch.load(checkpoint_path, map_location="cpu")["model"] + sd.pop("decoder.version") + return sd + + +def make_linear_from_emb(emb): + vocab_size, emb_size = emb.weight.shape + lin_layer = nn.Linear(vocab_size, emb_size, bias=False) + lin_layer.weight.data = emb.weight.data + return lin_layer + + +@torch.no_grad() +def convert_opt_checkpoint(checkpoint_path, pytorch_dump_folder_path, hf_checkpoint_name=None): + """ + Copy/paste/tweak model's weights to our BERT structure. + """ + state_dict = load_xsum_checkpoint(checkpoint_path) + + config = OPTConfig() + + model = OPTModel(config).eval() + model.load_state_dict(state_dict) + + # Check results + Path(pytorch_dump_folder_path).mkdir(exist_ok=True) + model.save_pretrained(pytorch_dump_folder_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + # Required parameters + parser.add_argument( + "--fairseq_path", type=str, help="opt.large, opt.large.cnn or a path to a model.pt on local filesystem." + ) + parser.add_argument("--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model.") + parser.add_argument( + "--hf_config", default=None, type=str, help="Which huggingface architecture to use: opt-large-xsum" + ) + args = parser.parse_args() + convert_opt_checkpoint(args.fairseq_path, args.pytorch_dump_folder_path, hf_checkpoint_name=args.hf_config) diff --git a/src/transformers/models/opt/modeling_flax_opt.py b/src/transformers/models/opt/modeling_flax_opt.py new file mode 100644 index 0000000000000..296f193d92dca --- /dev/null +++ b/src/transformers/models/opt/modeling_flax_opt.py @@ -0,0 +1,2021 @@ +# coding=utf-8 +# Copyright 2021 The Fairseq Authors and The Google Flax Team Authors And The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Flax OPT model.""" + +import math +import random +from functools import partial +from typing import Callable, Optional, Tuple + +import numpy as np + +import flax.linen as nn +import jax +import jax.numpy as jnp +from flax.core.frozen_dict import FrozenDict, freeze, unfreeze +from flax.linen import combine_masks, make_causal_mask +from flax.linen.attention import dot_product_attention_weights +from flax.traverse_util import flatten_dict, unflatten_dict +from jax import lax +from jax.random import PRNGKey + +from ...modeling_flax_outputs import ( + FlaxBaseModelOutput, + FlaxBaseModelOutputWithPastAndCrossAttentions, + FlaxCausalLMOutputWithCrossAttentions, + FlaxSeq2SeqLMOutput, + FlaxSeq2SeqModelOutput, + FlaxSeq2SeqQuestionAnsweringModelOutput, + FlaxSeq2SeqSequenceClassifierOutput, +) +from ...modeling_flax_utils import ( + ACT2FN, + FlaxPreTrainedModel, + append_call_sample_docstring, + append_replace_return_docstrings, + overwrite_call_docstring, +) +from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings +from .configuration_opt import OPTConfig + + +logger = logging.get_logger(__name__) + +_CHECKPOINT_FOR_DOC = "" +_CONFIG_FOR_DOC = "OPTConfig" +_TOKENIZER_FOR_DOC = "OPTTokenizer" + + +OPT_START_DOCSTRING = r""" + This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a Flax Linen + [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a + regular Flax Module and refer to the Flax documentation for all matter related to general usage and behavior. + + Finally, this model supports inherent JAX features such as: + + - [Just-In-Time (JIT) compilation](https://jax.readthedocs.io/en/latest/jax.html#just-in-time-compilation-jit) + - [Automatic Differentiation](https://jax.readthedocs.io/en/latest/jax.html#automatic-differentiation) + - [Vectorization](https://jax.readthedocs.io/en/latest/jax.html#vectorization-vmap) + - [Parallelization](https://jax.readthedocs.io/en/latest/jax.html#parallelization-pmap) + + Parameters: + config ([`OPTConfig`]): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights. + dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`): + The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and + `jax.numpy.bfloat16` (on TPUs). + + This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If + specified all the computation will be performed with the given `dtype`. + + **Note that this only specifies the dtype of the computation and does not influence the dtype of model + parameters.** + + If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and + [`~FlaxPreTrainedModel.to_bf16`]. +""" + +OPT_INPUTS_DOCSTRING = r""" + Args: + input_ids (`jnp.ndarray` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + + Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + For translation and summarization training, `decoder_input_ids` should be provided. If no + `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right + for denoising pre-training following the paper. + decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the + paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. + position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + decoder_position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the + range `[0, config.max_position_embeddings - 1]`. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + + +OPT_ENCODE_INPUTS_DOCSTRING = r""" + Args: + input_ids (`jnp.ndarray` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + + Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + +OPT_DECODE_INPUTS_DOCSTRING = r""" + Args: + decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + For translation and summarization training, `decoder_input_ids` should be provided. If no + `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right + for denoising pre-training following the paper. + encoder_outputs (`tuple(tuple(jnp.ndarray)`): + Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) + `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of + hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. + encoder_attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the + paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. + decoder_position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the + range `[0, config.max_position_embeddings - 1]`. + past_key_values (`Dict[str, np.ndarray]`, *optional*, returned by `init_cache` or when passing previous `past_key_values`): + Dictionary of pre-computed hidden-states (key and values in the attention blocks) that can be used for fast + auto-regressive decoding. Pre-computed key and value hidden-states are of shape *[batch_size, max_length]*. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + + +def shift_tokens_right(input_ids: np.array, pad_token_id: int, decoder_start_token_id: int) -> np.ndarray: + """ + Shift input ids one token to the right. + """ + shifted_input_ids = np.zeros_like(input_ids) + shifted_input_ids[:, 1:] = input_ids[:, :-1] + shifted_input_ids[:, 0] = decoder_start_token_id + + shifted_input_ids = np.where(shifted_input_ids == -100, pad_token_id, shifted_input_ids) + return shifted_input_ids + + +# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartAttention with Bart->OPT +class FlaxOPTAttention(nn.Module): + config: OPTConfig + embed_dim: int + num_heads: int + dropout: float = 0.0 + causal: bool = False + bias: bool = True + dtype: jnp.dtype = jnp.float32 # the dtype of the computation + + def setup(self) -> None: + self.head_dim = self.embed_dim // self.num_heads + if self.head_dim * self.num_heads != self.embed_dim: + raise ValueError( + f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}" + f" and `num_heads`: {self.num_heads})." + ) + + dense = partial( + nn.Dense, + self.embed_dim, + use_bias=self.bias, + dtype=self.dtype, + kernel_init=jax.nn.initializers.normal(self.config.init_std), + ) + + self.q_proj, self.k_proj, self.v_proj = dense(), dense(), dense() + self.out_proj = dense() + + self.dropout_layer = nn.Dropout(rate=self.dropout) + + if self.causal: + self.causal_mask = make_causal_mask( + jnp.ones((1, self.config.max_position_embeddings), dtype="bool"), dtype="bool" + ) + + def _split_heads(self, hidden_states): + return hidden_states.reshape(hidden_states.shape[:2] + (self.num_heads, self.head_dim)) + + def _merge_heads(self, hidden_states): + return hidden_states.reshape(hidden_states.shape[:2] + (self.embed_dim,)) + + @nn.compact + def _concatenate_to_cache(self, key, value, query, attention_mask): + """ + This function takes projected key, value states from a single input token and concatenates the states to cached + states from previous steps. This function is slighly adapted from the official Flax repository: + https://github.com/google/flax/blob/491ce18759622506588784b4fca0e4bf05f8c8cd/flax/linen/attention.py#L252 + """ + # detect if we're initializing by absence of existing cache data. + is_initialized = self.has_variable("cache", "cached_key") + cached_key = self.variable("cache", "cached_key", jnp.zeros, key.shape, key.dtype) + cached_value = self.variable("cache", "cached_value", jnp.zeros, value.shape, value.dtype) + cache_index = self.variable("cache", "cache_index", lambda: jnp.array(0, dtype=jnp.int32)) + + if is_initialized: + *batch_dims, max_length, num_heads, depth_per_head = cached_key.value.shape + # update key, value caches with our new 1d spatial slices + cur_index = cache_index.value + indices = (0,) * len(batch_dims) + (cur_index, 0, 0) + key = lax.dynamic_update_slice(cached_key.value, key, indices) + value = lax.dynamic_update_slice(cached_value.value, value, indices) + cached_key.value = key + cached_value.value = value + num_updated_cache_vectors = query.shape[1] + cache_index.value = cache_index.value + num_updated_cache_vectors + # causal mask for cached decoder self-attention: our single query position should only attend to those key positions that have already been generated and cached, not the remaining zero elements. + pad_mask = jnp.broadcast_to( + jnp.arange(max_length) < cur_index + num_updated_cache_vectors, + tuple(batch_dims) + (1, num_updated_cache_vectors, max_length), + ) + attention_mask = combine_masks(pad_mask, attention_mask) + return key, value, attention_mask + + def __call__( + self, + hidden_states: jnp.ndarray, + key_value_states: Optional[jnp.ndarray] = None, + attention_mask: Optional[jnp.ndarray] = None, + init_cache: bool = False, + deterministic: bool = True, + ) -> Tuple[jnp.ndarray]: + """Input shape: Batch x Time x Channel""" + + # if key_value_states are provided this layer is used as a cross-attention layer + # for the decoder + is_cross_attention = key_value_states is not None + batch_size = hidden_states.shape[0] + + # get query proj + query_states = self.q_proj(hidden_states) + # get key, value proj + if is_cross_attention: + # cross_attentions + key_states = self.k_proj(key_value_states) + value_states = self.v_proj(key_value_states) + else: + # self_attention + key_states = self.k_proj(hidden_states) + value_states = self.v_proj(hidden_states) + + query_states = self._split_heads(query_states) + key_states = self._split_heads(key_states) + value_states = self._split_heads(value_states) + + # handle cache prepare causal attention mask + if self.causal: + query_length, key_length = query_states.shape[1], key_states.shape[1] + if self.has_variable("cache", "cached_key"): + mask_shift = self.variables["cache"]["cache_index"] + max_decoder_length = self.variables["cache"]["cached_key"].shape[1] + causal_mask = lax.dynamic_slice( + self.causal_mask, (0, 0, mask_shift, 0), (1, 1, query_length, max_decoder_length) + ) + else: + causal_mask = self.causal_mask[:, :, :query_length, :key_length] + causal_mask = jnp.broadcast_to(causal_mask, (batch_size,) + causal_mask.shape[1:]) + + # combine masks if needed + if attention_mask is not None and self.causal: + attention_mask = jnp.broadcast_to(jnp.expand_dims(attention_mask, axis=(-3, -2)), causal_mask.shape) + attention_mask = combine_masks(attention_mask, causal_mask) + elif self.causal: + attention_mask = causal_mask + elif attention_mask is not None: + attention_mask = jnp.expand_dims(attention_mask, axis=(-3, -2)) + + # During fast autoregressive decoding, we feed one position at a time, + # and cache the keys and values step by step. + if self.causal and (self.has_variable("cache", "cached_key") or init_cache): + key_states, value_states, attention_mask = self._concatenate_to_cache( + key_states, value_states, query_states, attention_mask + ) + + # Convert the boolean attention mask to an attention bias. + if attention_mask is not None: + # attention mask in the form of attention bias + attention_bias = lax.select( + attention_mask > 0, + jnp.full(attention_mask.shape, 0.0).astype(self.dtype), + jnp.full(attention_mask.shape, float("-inf")).astype(self.dtype), + ) + else: + attention_bias = None + + dropout_rng = None + if not deterministic and self.dropout > 0.0: + dropout_rng = self.make_rng("dropout") + + attn_weights = dot_product_attention_weights( + query_states, + key_states, + bias=attention_bias, + dropout_rng=dropout_rng, + dropout_rate=self.dropout, + broadcast_dropout=True, + deterministic=deterministic, + dtype=self.dtype, + precision=None, + ) + + attn_output = jnp.einsum("...hqk,...khd->...qhd", attn_weights, value_states) + attn_output = self._merge_heads(attn_output) + attn_output = self.out_proj(attn_output) + + return attn_output, attn_weights + + +# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartEncoderLayer with Bart->OPT +class FlaxOPTEncoderLayer(nn.Module): + config: OPTConfig + dtype: jnp.dtype = jnp.float32 + + def setup(self) -> None: + self.embed_dim = self.config.d_model + self.self_attn = FlaxOPTAttention( + config=self.config, + embed_dim=self.embed_dim, + num_heads=self.config.encoder_attention_heads, + dropout=self.config.attention_dropout, + dtype=self.dtype, + ) + self.self_attn_layer_norm = nn.LayerNorm(dtype=self.dtype, epsilon=1e-05) + self.dropout_layer = nn.Dropout(rate=self.config.dropout) + self.activation_fn = ACT2FN[self.config.activation_function] + self.activation_dropout_layer = nn.Dropout(rate=self.config.activation_dropout) + self.fc1 = nn.Dense( + self.config.encoder_ffn_dim, + dtype=self.dtype, + kernel_init=jax.nn.initializers.normal(self.config.init_std), + ) + self.fc2 = nn.Dense( + self.embed_dim, dtype=self.dtype, kernel_init=jax.nn.initializers.normal(self.config.init_std) + ) + self.final_layer_norm = nn.LayerNorm(dtype=self.dtype, epsilon=1e-05) + + def __call__( + self, + hidden_states: jnp.ndarray, + attention_mask: jnp.ndarray, + output_attentions: bool = True, + deterministic: bool = True, + ) -> Tuple[jnp.ndarray]: + residual = hidden_states + hidden_states, attn_weights = self.self_attn(hidden_states=hidden_states, attention_mask=attention_mask) + + hidden_states = self.dropout_layer(hidden_states, deterministic=deterministic) + hidden_states = residual + hidden_states + hidden_states = self.self_attn_layer_norm(hidden_states) + + residual = hidden_states + hidden_states = self.activation_fn(self.fc1(hidden_states)) + hidden_states = self.activation_dropout_layer(hidden_states, deterministic=deterministic) + hidden_states = self.fc2(hidden_states) + hidden_states = self.dropout_layer(hidden_states, deterministic=deterministic) + hidden_states = residual + hidden_states + hidden_states = self.final_layer_norm(hidden_states) + + outputs = (hidden_states,) + + if output_attentions: + outputs += (attn_weights,) + + return outputs + + +# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartEncoderLayerCollection with Bart->OPT +class FlaxOPTEncoderLayerCollection(nn.Module): + config: OPTConfig + dtype: jnp.dtype = jnp.float32 # the dtype of the computation + + def setup(self): + self.layers = [ + FlaxOPTEncoderLayer(self.config, name=str(i), dtype=self.dtype) for i in range(self.config.encoder_layers) + ] + self.layerdrop = self.config.encoder_layerdrop + + def __call__( + self, + hidden_states, + attention_mask, + deterministic: bool = True, + output_attentions: bool = False, + output_hidden_states: bool = False, + return_dict: bool = True, + ): + all_attentions = () if output_attentions else None + all_hidden_states = () if output_hidden_states else None + + for encoder_layer in self.layers: + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description) + dropout_probability = random.uniform(0, 1) + if not deterministic and (dropout_probability < self.layerdrop): # skip the layer + layer_outputs = (None, None) + else: + layer_outputs = encoder_layer( + hidden_states, + attention_mask, + output_attentions, + deterministic, + ) + hidden_states = layer_outputs[0] + if output_attentions: + all_attentions = all_attentions + (layer_outputs[1],) + + if output_hidden_states: + all_hidden_states += (hidden_states,) + + outputs = (hidden_states, all_hidden_states, all_attentions) + + if not return_dict: + return tuple(v for v in outputs if v is not None) + + return FlaxBaseModelOutput( + last_hidden_state=hidden_states, hidden_states=all_hidden_states, attentions=all_attentions + ) + + +# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartDecoderLayer with Bart->OPT +class FlaxOPTDecoderLayer(nn.Module): + config: OPTConfig + dtype: jnp.dtype = jnp.float32 + + def setup(self) -> None: + self.embed_dim = self.config.d_model + self.self_attn = FlaxOPTAttention( + config=self.config, + embed_dim=self.embed_dim, + num_heads=self.config.decoder_attention_heads, + dropout=self.config.attention_dropout, + causal=True, + dtype=self.dtype, + ) + self.dropout_layer = nn.Dropout(rate=self.config.dropout) + self.activation_fn = ACT2FN[self.config.activation_function] + self.activation_dropout_layer = nn.Dropout(rate=self.config.activation_dropout) + + self.self_attn_layer_norm = nn.LayerNorm(dtype=self.dtype, epsilon=1e-05) + self.encoder_attn = FlaxOPTAttention( + config=self.config, + embed_dim=self.embed_dim, + num_heads=self.config.decoder_attention_heads, + dropout=self.config.attention_dropout, + dtype=self.dtype, + ) + self.encoder_attn_layer_norm = nn.LayerNorm(dtype=self.dtype, epsilon=1e-05) + self.fc1 = nn.Dense( + self.config.encoder_ffn_dim, + dtype=self.dtype, + kernel_init=jax.nn.initializers.normal(self.config.init_std), + ) + self.fc2 = nn.Dense( + self.embed_dim, dtype=self.dtype, kernel_init=jax.nn.initializers.normal(self.config.init_std) + ) + self.final_layer_norm = nn.LayerNorm(dtype=self.dtype, epsilon=1e-05) + + def __call__( + self, + hidden_states: jnp.ndarray, + attention_mask: jnp.ndarray, + encoder_hidden_states: Optional[jnp.ndarray] = None, + encoder_attention_mask: Optional[jnp.ndarray] = None, + init_cache: bool = False, + output_attentions: bool = True, + deterministic: bool = True, + ) -> Tuple[jnp.ndarray]: + residual = hidden_states + + # Self Attention + hidden_states, self_attn_weights = self.self_attn( + hidden_states=hidden_states, attention_mask=attention_mask, init_cache=init_cache + ) + hidden_states = self.dropout_layer(hidden_states, deterministic=deterministic) + hidden_states = residual + hidden_states + hidden_states = self.self_attn_layer_norm(hidden_states) + + # Cross-Attention Block + cross_attn_weights = None + if encoder_hidden_states is not None: + residual = hidden_states + + hidden_states, cross_attn_weights = self.encoder_attn( + hidden_states=hidden_states, + key_value_states=encoder_hidden_states, + attention_mask=encoder_attention_mask, + ) + hidden_states = self.dropout_layer(hidden_states, deterministic=deterministic) + hidden_states = residual + hidden_states + hidden_states = self.encoder_attn_layer_norm(hidden_states) + + # Fully Connected + residual = hidden_states + hidden_states = self.activation_fn(self.fc1(hidden_states)) + hidden_states = self.activation_dropout_layer(hidden_states, deterministic=deterministic) + hidden_states = self.fc2(hidden_states) + hidden_states = self.dropout_layer(hidden_states, deterministic=deterministic) + hidden_states = residual + hidden_states + hidden_states = self.final_layer_norm(hidden_states) + + outputs = (hidden_states,) + + if output_attentions: + outputs += (self_attn_weights, cross_attn_weights) + + return outputs + + +# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartDecoderLayerCollection with Bart->OPT +class FlaxOPTDecoderLayerCollection(nn.Module): + config: OPTConfig + dtype: jnp.dtype = jnp.float32 # the dtype of the computation + + def setup(self): + self.layers = [ + FlaxOPTDecoderLayer(self.config, name=str(i), dtype=self.dtype) for i in range(self.config.decoder_layers) + ] + self.layerdrop = self.config.decoder_layerdrop + + def __call__( + self, + hidden_states, + attention_mask, + encoder_hidden_states: Optional[jnp.ndarray] = None, + encoder_attention_mask: Optional[jnp.ndarray] = None, + deterministic: bool = True, + init_cache: bool = False, + output_attentions: bool = False, + output_hidden_states: bool = False, + return_dict: bool = True, + ): + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + all_cross_attentions = () if (output_attentions and encoder_hidden_states is not None) else None + + for decoder_layer in self.layers: + if output_hidden_states: + all_hidden_states += (hidden_states,) + # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description) + dropout_probability = random.uniform(0, 1) + if not deterministic and (dropout_probability < self.layerdrop): + layer_outputs = (None, None, None) + else: + layer_outputs = decoder_layer( + hidden_states, + attention_mask=attention_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + init_cache=init_cache, + output_attentions=output_attentions, + deterministic=deterministic, + ) + + hidden_states = layer_outputs[0] + if output_attentions: + all_self_attns += (layer_outputs[1],) + + if encoder_hidden_states is not None: + all_cross_attentions += (layer_outputs[2],) + + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + outputs = [hidden_states, all_hidden_states, all_self_attns, all_cross_attentions] + + if not return_dict: + return tuple(v for v in outputs if v is not None) + + return FlaxBaseModelOutputWithPastAndCrossAttentions( + last_hidden_state=hidden_states, + hidden_states=all_hidden_states, + attentions=all_self_attns, + cross_attentions=all_cross_attentions, + ) + + +# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartClassificationHead with Bart->OPT +class FlaxOPTClassificationHead(nn.Module): + """Head for sentence-level classification tasks.""" + + config: OPTConfig + inner_dim: int + num_classes: int + pooler_dropout: float + dtype: jnp.dtype = jnp.float32 + + def setup(self): + self.dense = nn.Dense( + self.inner_dim, dtype=self.dtype, kernel_init=jax.nn.initializers.normal(self.config.init_std) + ) + self.dropout = nn.Dropout(rate=self.pooler_dropout) + self.out_proj = nn.Dense( + self.num_classes, + dtype=self.dtype, + kernel_init=jax.nn.initializers.normal(self.config.init_std), + ) + + def __call__(self, hidden_states: jnp.ndarray, deterministic: bool): + hidden_states = self.dropout(hidden_states, deterministic=deterministic) + hidden_states = self.dense(hidden_states) + hidden_states = jnp.tanh(hidden_states) + hidden_states = self.dropout(hidden_states, deterministic=deterministic) + hidden_states = self.out_proj(hidden_states) + return hidden_states + + +# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartEncoder with Bart->OPT +class FlaxOPTEncoder(nn.Module): + config: OPTConfig + embed_tokens: nn.Embed + dtype: jnp.dtype = jnp.float32 # the dtype of the computation + + def setup(self): + self.dropout_layer = nn.Dropout(rate=self.config.dropout) + + embed_dim = self.config.d_model + self.padding_idx = self.config.pad_token_id + self.max_source_positions = self.config.max_position_embeddings + self.embed_scale = math.sqrt(embed_dim) if self.config.scale_embedding else 1.0 + + # OPT is set up so that if padding_idx is specified then offset the embedding ids by 2 + # and adjust num_embeddings appropriately. Other models don't have this hack + self.offset = 2 + self.embed_positions = nn.Embed( + self.config.max_position_embeddings + self.offset, + embed_dim, + embedding_init=jax.nn.initializers.normal(self.config.init_std), + ) + self.layers = FlaxOPTEncoderLayerCollection(self.config, self.dtype) + self.layernorm_embedding = nn.LayerNorm(dtype=self.dtype, epsilon=1e-05) + + def __call__( + self, + input_ids, + attention_mask, + position_ids, + output_attentions: bool = False, + output_hidden_states: bool = False, + return_dict: bool = True, + deterministic: bool = True, + ): + input_shape = input_ids.shape + input_ids = input_ids.reshape(-1, input_shape[-1]) + + inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale + + embed_pos = self.embed_positions(position_ids + self.offset) + + hidden_states = inputs_embeds + embed_pos + hidden_states = self.layernorm_embedding(hidden_states) + hidden_states = self.dropout_layer(hidden_states, deterministic=deterministic) + + outputs = self.layers( + hidden_states, + attention_mask, + deterministic=deterministic, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + if not return_dict: + return outputs + + return FlaxBaseModelOutput( + last_hidden_state=outputs.last_hidden_state, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + + +# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartDecoder with Bart->OPT +class FlaxOPTDecoder(nn.Module): + config: OPTConfig + embed_tokens: nn.Embed + dtype: jnp.dtype = jnp.float32 # the dtype of the computation + + def setup(self): + self.dropout_layer = nn.Dropout(rate=self.config.dropout) + + embed_dim = self.config.d_model + self.padding_idx = self.config.pad_token_id + self.max_target_positions = self.config.max_position_embeddings + self.embed_scale = math.sqrt(self.config.d_model) if self.config.scale_embedding else 1.0 + + # OPT is set up so that if padding_idx is specified then offset the embedding ids by 2 + # and adjust num_embeddings appropriately. Other models don't have this hack + self.offset = 2 + self.embed_positions = nn.Embed( + self.config.max_position_embeddings + self.offset, + embed_dim, + embedding_init=jax.nn.initializers.normal(self.config.init_std), + ) + + self.layers = FlaxOPTDecoderLayerCollection(self.config, self.dtype) + self.layernorm_embedding = nn.LayerNorm(dtype=self.dtype, epsilon=1e-05) + + def __call__( + self, + input_ids, + attention_mask, + position_ids, + encoder_hidden_states: Optional[jnp.ndarray] = None, + encoder_attention_mask: Optional[jnp.ndarray] = None, + init_cache: bool = False, + output_attentions: bool = False, + output_hidden_states: bool = False, + return_dict: bool = True, + deterministic: bool = True, + ): + input_shape = input_ids.shape + input_ids = input_ids.reshape(-1, input_shape[-1]) + + inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale + + # embed positions + positions = self.embed_positions(position_ids + self.offset) + + hidden_states = inputs_embeds + positions + hidden_states = self.layernorm_embedding(hidden_states) + + hidden_states = self.dropout_layer(hidden_states, deterministic=deterministic) + + outputs = self.layers( + hidden_states, + attention_mask, + encoder_hidden_states, + encoder_attention_mask, + deterministic=deterministic, + init_cache=init_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + if not return_dict: + return outputs + + return FlaxBaseModelOutputWithPastAndCrossAttentions( + last_hidden_state=outputs.last_hidden_state, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + cross_attentions=outputs.cross_attentions, + ) + + +# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartModule with Bart->OPT +class FlaxOPTModule(nn.Module): + config: OPTConfig + dtype: jnp.dtype = jnp.float32 # the dtype of the computation + + def setup(self): + self.shared = nn.Embed( + self.config.vocab_size, + self.config.d_model, + embedding_init=jax.nn.initializers.normal(self.config.init_std), + ) + + self.encoder = FlaxOPTEncoder(self.config, dtype=self.dtype, embed_tokens=self.shared) + self.decoder = FlaxOPTDecoder(self.config, dtype=self.dtype, embed_tokens=self.shared) + + def _get_encoder_module(self): + return self.encoder + + def _get_decoder_module(self): + return self.decoder + + def __call__( + self, + input_ids, + attention_mask, + decoder_input_ids, + decoder_attention_mask, + position_ids, + decoder_position_ids, + output_attentions: bool = False, + output_hidden_states: bool = False, + return_dict: bool = True, + deterministic: bool = True, + ): + encoder_outputs = self.encoder( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + deterministic=deterministic, + ) + + decoder_outputs = self.decoder( + input_ids=decoder_input_ids, + attention_mask=decoder_attention_mask, + position_ids=decoder_position_ids, + encoder_hidden_states=encoder_outputs[0], + encoder_attention_mask=attention_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + deterministic=deterministic, + ) + + if not return_dict: + return decoder_outputs + encoder_outputs + + return FlaxSeq2SeqModelOutput( + last_hidden_state=decoder_outputs.last_hidden_state, + decoder_hidden_states=decoder_outputs.hidden_states, + decoder_attentions=decoder_outputs.attentions, + cross_attentions=decoder_outputs.cross_attentions, + encoder_last_hidden_state=encoder_outputs.last_hidden_state, + encoder_hidden_states=encoder_outputs.hidden_states, + encoder_attentions=encoder_outputs.attentions, + ) + + +# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartPreTrainedModel with BART->OPT,Bart->OPT,bart->opt +class FlaxOPTPreTrainedModel(FlaxPreTrainedModel): + config_class = OPTConfig + base_model_prefix: str = "model" + module_class: nn.Module = None + + def __init__( + self, + config: OPTConfig, + input_shape: Tuple[int] = (1, 1), + seed: int = 0, + dtype: jnp.dtype = jnp.float32, + _do_init: bool = True, + **kwargs + ): + module = self.module_class(config=config, dtype=dtype, **kwargs) + super().__init__(config, module, input_shape=input_shape, seed=seed, dtype=dtype, _do_init=_do_init) + + def init_weights(self, rng: jax.random.PRNGKey, input_shape: Tuple, params: FrozenDict = None) -> FrozenDict: + # init input tensors + input_ids = jnp.zeros(input_shape, dtype="i4") + # make sure initialization pass will work for FlaxOPTForSequenceClassificationModule + input_ids = input_ids.at[(..., -1)].set(self.config.eos_token_id) + attention_mask = jnp.ones_like(input_ids) + decoder_input_ids = input_ids + decoder_attention_mask = jnp.ones_like(input_ids) + + batch_size, sequence_length = input_ids.shape + position_ids = jnp.broadcast_to(jnp.arange(sequence_length)[None, :], (batch_size, sequence_length)) + decoder_position_ids = jnp.broadcast_to(jnp.arange(sequence_length)[None, :], (batch_size, sequence_length)) + + params_rng, dropout_rng = jax.random.split(rng) + rngs = {"params": params_rng, "dropout": dropout_rng} + + random_params = self.module.init( + rngs, + input_ids, + attention_mask, + decoder_input_ids, + decoder_attention_mask, + position_ids, + decoder_position_ids, + )["params"] + + if params is not None: + random_params = flatten_dict(unfreeze(random_params)) + params = flatten_dict(unfreeze(params)) + for missing_key in self._missing_keys: + params[missing_key] = random_params[missing_key] + self._missing_keys = set() + return freeze(unflatten_dict(params)) + else: + return random_params + + def init_cache(self, batch_size, max_length, encoder_outputs): + r""" + Args: + batch_size (`int`): + batch_size used for fast auto-regressive decoding. Defines the batch size of the initialized cache. + max_length (`int`): + maximum possible length for auto-regressive decoding. Defines the sequence length of the initialized + cache. + encoder_outputs (`Union[FlaxBaseModelOutput, tuple(tuple(jnp.ndarray)]`): + `encoder_outputs` consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: + `attentions`). `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) + is a sequence of hidden-states at the output of the last layer of the encoder. Used in the + cross-attention of the decoder. + """ + # init input variables to retrieve cache + decoder_input_ids = jnp.ones((batch_size, max_length), dtype="i4") + decoder_attention_mask = jnp.ones_like(decoder_input_ids) + decoder_position_ids = jnp.broadcast_to( + jnp.arange(jnp.atleast_2d(decoder_input_ids).shape[-1]), decoder_input_ids.shape + ) + + def _decoder_forward(module, decoder_input_ids, decoder_attention_mask, decoder_position_ids, **kwargs): + decoder_module = module._get_decoder_module() + return decoder_module( + decoder_input_ids, + decoder_attention_mask, + decoder_position_ids, + **kwargs, + ) + + init_variables = self.module.init( + jax.random.PRNGKey(0), + decoder_input_ids=decoder_input_ids, + decoder_attention_mask=decoder_attention_mask, + decoder_position_ids=decoder_position_ids, + encoder_hidden_states=encoder_outputs[0], + init_cache=True, + method=_decoder_forward, # we only need to call the decoder to init the cache + ) + return unfreeze(init_variables["cache"]) + + @add_start_docstrings(OPT_ENCODE_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=FlaxBaseModelOutput, config_class=OPTConfig) + def encode( + self, + input_ids: jnp.ndarray, + attention_mask: Optional[jnp.ndarray] = None, + position_ids: Optional[jnp.ndarray] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + train: bool = False, + params: dict = None, + dropout_rng: PRNGKey = None, + ): + r""" + Returns: + + Example: + + ```python + >>> from transformers import OPTTokenizer, FlaxOPTForConditionalGeneration + + >>> model = FlaxOPTForConditionalGeneration.from_pretrained("facebook/opt-large-cnn") + >>> tokenizer = OPTTokenizer.from_pretrained("facebook/opt-large-cnn") + + >>> text = "My friends are cool but they eat too many carbs." + >>> inputs = tokenizer(text, max_length=1024, return_tensors="jax") + >>> encoder_outputs = model.encode(**inputs) + ```""" + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.return_dict + + if attention_mask is None: + attention_mask = jnp.ones_like(input_ids) + if position_ids is None: + batch_size, sequence_length = input_ids.shape + position_ids = jnp.broadcast_to(jnp.arange(sequence_length)[None, :], (batch_size, sequence_length)) + + # Handle any PRNG if needed + rngs = {} + if dropout_rng is not None: + rngs["dropout"] = dropout_rng + + def _encoder_forward(module, input_ids, attention_mask, position_ids, **kwargs): + encode_module = module._get_encoder_module() + return encode_module(input_ids, attention_mask, position_ids, **kwargs) + + return self.module.apply( + {"params": params or self.params}, + input_ids=jnp.array(input_ids, dtype="i4"), + attention_mask=jnp.array(attention_mask, dtype="i4"), + position_ids=jnp.array(position_ids, dtype="i4"), + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + deterministic=not train, + rngs=rngs, + method=_encoder_forward, + ) + + @add_start_docstrings(OPT_DECODE_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=FlaxBaseModelOutputWithPastAndCrossAttentions, config_class=OPTConfig) + def decode( + self, + decoder_input_ids, + encoder_outputs, + encoder_attention_mask: Optional[jnp.ndarray] = None, + decoder_attention_mask: Optional[jnp.ndarray] = None, + decoder_position_ids: Optional[jnp.ndarray] = None, + past_key_values: dict = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + train: bool = False, + params: dict = None, + dropout_rng: PRNGKey = None, + ): + r""" + Returns: + + Example: + + ```python + >>> import jax.numpy as jnp + >>> from transformers import OPTTokenizer, FlaxOPTForConditionalGeneration + + >>> model = FlaxOPTForConditionalGeneration.from_pretrained("facebook/opt-large-cnn") + >>> tokenizer = OPTTokenizer.from_pretrained("facebook/opt-large-cnn") + + >>> text = "My friends are cool but they eat too many carbs." + >>> inputs = tokenizer(text, max_length=1024, return_tensors="jax") + >>> encoder_outputs = model.encode(**inputs) + + >>> decoder_start_token_id = model.config.decoder_start_token_id + >>> decoder_input_ids = jnp.ones((inputs.input_ids.shape[0], 1), dtype="i4") * decoder_start_token_id + + >>> outputs = model.decode(decoder_input_ids, encoder_outputs) + >>> last_decoder_hidden_states = outputs.last_hidden_state + ```""" + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.return_dict + + encoder_hidden_states = encoder_outputs[0] + if encoder_attention_mask is None: + batch_size, sequence_length = encoder_hidden_states.shape[:2] + encoder_attention_mask = jnp.ones((batch_size, sequence_length)) + + batch_size, sequence_length = decoder_input_ids.shape + if decoder_attention_mask is None: + decoder_attention_mask = jnp.ones((batch_size, sequence_length)) + + if decoder_position_ids is None: + if past_key_values is not None: + raise ValueError("Make sure to provide `decoder_position_ids` when passing `past_key_values`.") + + decoder_position_ids = jnp.broadcast_to( + jnp.arange(sequence_length)[None, :], (batch_size, sequence_length) + ) + + # Handle any PRNG if needed + rngs = {} + if dropout_rng is not None: + rngs["dropout"] = dropout_rng + + inputs = {"params": params or self.params} + + # if past_key_values are passed then cache is already initialized a private flag init_cache has to be + # passed down to ensure cache is used. It has to be made sure that cache is marked as mutable so that + # it can be changed by FlaxOPTAttention module + if past_key_values: + inputs["cache"] = past_key_values + mutable = ["cache"] + else: + mutable = False + + def _decoder_forward(module, decoder_input_ids, decoder_attention_mask, decoder_position_ids, **kwargs): + decoder_module = module._get_decoder_module() + return decoder_module( + decoder_input_ids, + decoder_attention_mask, + decoder_position_ids, + **kwargs, + ) + + outputs = self.module.apply( + inputs, + decoder_input_ids=jnp.array(decoder_input_ids, dtype="i4"), + decoder_attention_mask=jnp.array(decoder_attention_mask, dtype="i4"), + decoder_position_ids=jnp.array(decoder_position_ids, dtype="i4"), + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=jnp.array(encoder_attention_mask, dtype="i4"), + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + deterministic=not train, + rngs=rngs, + mutable=mutable, + method=_decoder_forward, + ) + + # add updated cache to model output + if past_key_values is not None and return_dict: + outputs, past = outputs + outputs["past_key_values"] = unfreeze(past["cache"]) + return outputs + elif past_key_values is not None and not return_dict: + outputs, past = outputs + outputs = outputs[:1] + (unfreeze(past["cache"]),) + outputs[1:] + + return outputs + + @add_start_docstrings_to_model_forward(OPT_INPUTS_DOCSTRING) + def __call__( + self, + input_ids: jnp.ndarray, + attention_mask: Optional[jnp.ndarray] = None, + decoder_input_ids: Optional[jnp.ndarray] = None, + decoder_attention_mask: Optional[jnp.ndarray] = None, + position_ids: Optional[jnp.ndarray] = None, + decoder_position_ids: Optional[jnp.ndarray] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + train: bool = False, + params: dict = None, + dropout_rng: PRNGKey = None, + ): + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.return_dict + + # prepare encoder inputs + if attention_mask is None: + attention_mask = jnp.ones_like(input_ids) + if position_ids is None: + batch_size, sequence_length = input_ids.shape + position_ids = jnp.broadcast_to(jnp.arange(sequence_length)[None, :], (batch_size, sequence_length)) + + # prepare decoder inputs + if decoder_input_ids is None: + decoder_input_ids = shift_tokens_right( + input_ids, self.config.pad_token_id, decoder_start_token_id=self.config.decoder_start_token_id + ) + if decoder_attention_mask is None: + decoder_attention_mask = jnp.ones_like(decoder_input_ids) + if decoder_position_ids is None: + batch_size, sequence_length = decoder_input_ids.shape + decoder_position_ids = jnp.broadcast_to( + jnp.arange(sequence_length)[None, :], (batch_size, sequence_length) + ) + + # Handle any PRNG if needed + rngs = {"dropout": dropout_rng} if dropout_rng is not None else {} + + return self.module.apply( + {"params": params or self.params}, + input_ids=jnp.array(input_ids, dtype="i4"), + attention_mask=jnp.array(attention_mask, dtype="i4"), + position_ids=jnp.array(position_ids, dtype="i4"), + decoder_input_ids=jnp.array(decoder_input_ids, dtype="i4"), + decoder_attention_mask=jnp.array(decoder_attention_mask, dtype="i4"), + decoder_position_ids=jnp.array(decoder_position_ids, dtype="i4"), + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + deterministic=not train, + rngs=rngs, + ) + + +@add_start_docstrings( + "The bare OPT Model transformer outputting raw hidden-states without any specific head on top.", + OPT_START_DOCSTRING, +) +# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartModel with Bart->OPT +class FlaxOPTModel(FlaxOPTPreTrainedModel): + config: OPTConfig + dtype: jnp.dtype = jnp.float32 # the dtype of the computation + module_class = FlaxOPTModule + + +append_call_sample_docstring( + FlaxOPTModel, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSeq2SeqModelOutput, _CONFIG_FOR_DOC +) + + +# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartForConditionalGenerationModule with Bart->OPT +class FlaxOPTForConditionalGenerationModule(nn.Module): + config: OPTConfig + dtype: jnp.dtype = jnp.float32 + bias_init: Callable[..., jnp.ndarray] = jax.nn.initializers.zeros + + def setup(self): + self.model = FlaxOPTModule(config=self.config, dtype=self.dtype) + self.lm_head = nn.Dense( + self.model.shared.num_embeddings, + use_bias=False, + dtype=self.dtype, + kernel_init=jax.nn.initializers.normal(self.config.init_std), + ) + self.final_logits_bias = self.param("final_logits_bias", self.bias_init, (1, self.model.shared.num_embeddings)) + + def _get_encoder_module(self): + return self.model.encoder + + def _get_decoder_module(self): + return self.model.decoder + + def __call__( + self, + input_ids, + attention_mask, + decoder_input_ids, + decoder_attention_mask, + position_ids, + decoder_position_ids, + output_attentions: bool = False, + output_hidden_states: bool = False, + return_dict: bool = True, + deterministic: bool = True, + ): + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + decoder_input_ids=decoder_input_ids, + decoder_attention_mask=decoder_attention_mask, + position_ids=position_ids, + decoder_position_ids=decoder_position_ids, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + deterministic=deterministic, + ) + + hidden_states = outputs[0] + + if self.config.tie_word_embeddings: + shared_embedding = self.model.variables["params"]["shared"]["embedding"] + lm_logits = self.lm_head.apply({"params": {"kernel": shared_embedding.T}}, hidden_states) + else: + lm_logits = self.lm_head(hidden_states) + + lm_logits += jax.lax.stop_gradient(self.final_logits_bias.astype(self.dtype)) + + if not return_dict: + output = (lm_logits,) + outputs[1:] + return output + + return FlaxSeq2SeqLMOutput( + logits=lm_logits, + decoder_hidden_states=outputs.decoder_hidden_states, + decoder_attentions=outputs.decoder_attentions, + cross_attentions=outputs.cross_attentions, + encoder_last_hidden_state=outputs.encoder_last_hidden_state, + encoder_hidden_states=outputs.encoder_hidden_states, + encoder_attentions=outputs.encoder_attentions, + ) + + +@add_start_docstrings( + "The OPT Model with a language modeling head. Can be used for summarization.", OPT_START_DOCSTRING +) +# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartForConditionalGeneration with BART->OPT,Bart->OPT,bart->opt +class FlaxOPTForConditionalGeneration(FlaxOPTPreTrainedModel): + module_class = FlaxOPTForConditionalGenerationModule + dtype: jnp.dtype = jnp.float32 + + @add_start_docstrings(OPT_DECODE_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=FlaxCausalLMOutputWithCrossAttentions, config_class=OPTConfig) + def decode( + self, + decoder_input_ids, + encoder_outputs, + encoder_attention_mask: Optional[jnp.ndarray] = None, + decoder_attention_mask: Optional[jnp.ndarray] = None, + decoder_position_ids: Optional[jnp.ndarray] = None, + past_key_values: dict = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + train: bool = False, + params: dict = None, + dropout_rng: PRNGKey = None, + ): + r""" + Returns: + + Example: + + ```python + >>> import jax.numpy as jnp + >>> from transformers import OPTTokenizer, FlaxOPTForConditionalGeneration + + >>> model = FlaxOPTForConditionalGeneration.from_pretrained("facebook/opt-large-cnn") + >>> tokenizer = OPTTokenizer.from_pretrained("facebook/opt-large-cnn") + + >>> text = "My friends are cool but they eat too many carbs." + >>> inputs = tokenizer(text, max_length=1024, return_tensors="jax") + >>> encoder_outputs = model.encode(**inputs) + + >>> decoder_start_token_id = model.config.decoder_start_token_id + >>> decoder_input_ids = jnp.ones((inputs.input_ids.shape[0], 1), dtype="i4") * decoder_start_token_id + + >>> outputs = model.decode(decoder_input_ids, encoder_outputs) + >>> logits = outputs.logits + ```""" + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.return_dict + + encoder_hidden_states = encoder_outputs[0] + if encoder_attention_mask is None: + batch_size, sequence_length = encoder_hidden_states.shape[:2] + encoder_attention_mask = jnp.ones((batch_size, sequence_length)) + + batch_size, sequence_length = decoder_input_ids.shape + if decoder_attention_mask is None: + decoder_attention_mask = jnp.ones((batch_size, sequence_length)) + + if decoder_position_ids is None: + if past_key_values is not None: + raise ValueError("Make sure to provide `decoder_position_ids` when passing `past_key_values`.") + + decoder_position_ids = jnp.broadcast_to( + jnp.arange(sequence_length)[None, :], (batch_size, sequence_length) + ) + + # Handle any PRNG if needed + rngs = {} + if dropout_rng is not None: + rngs["dropout"] = dropout_rng + + inputs = {"params": params or self.params} + + # if past_key_values are passed then cache is already initialized a private flag init_cache has to be + # passed down to ensure cache is used. It has to be made sure that cache is marked as mutable so that + # it can be changed by FlaxOPTAttention module + if past_key_values: + inputs["cache"] = past_key_values + mutable = ["cache"] + else: + mutable = False + + def _decoder_forward(module, decoder_input_ids, decoder_attention_mask, decoder_position_ids, **kwargs): + decoder_module = module._get_decoder_module() + outputs = decoder_module( + decoder_input_ids, + decoder_attention_mask, + decoder_position_ids, + **kwargs, + ) + hidden_states = outputs[0] + + if self.config.tie_word_embeddings: + shared_embedding = module.model.variables["params"]["shared"]["embedding"] + lm_logits = module.lm_head.apply({"params": {"kernel": shared_embedding.T}}, hidden_states) + else: + lm_logits = module.lm_head(hidden_states) + + lm_logits += module.final_logits_bias.astype(self.dtype) + return lm_logits, outputs + + outputs = self.module.apply( + inputs, + decoder_input_ids=jnp.array(decoder_input_ids, dtype="i4"), + decoder_attention_mask=jnp.array(decoder_attention_mask, dtype="i4"), + decoder_position_ids=jnp.array(decoder_position_ids, dtype="i4"), + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=jnp.array(encoder_attention_mask, dtype="i4"), + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + deterministic=not train, + rngs=rngs, + mutable=mutable, + method=_decoder_forward, + ) + + if past_key_values is None: + lm_logits, decoder_outputs = outputs + else: + (lm_logits, decoder_outputs), past = outputs + + if return_dict: + outputs = FlaxCausalLMOutputWithCrossAttentions( + logits=lm_logits, + hidden_states=decoder_outputs.hidden_states, + attentions=decoder_outputs.attentions, + cross_attentions=decoder_outputs.cross_attentions, + ) + else: + outputs = (lm_logits,) + decoder_outputs[1:] + + # add updated cache to model output + if past_key_values is not None and return_dict: + outputs["past_key_values"] = unfreeze(past["cache"]) + return outputs + elif past_key_values is not None and not return_dict: + outputs = outputs[:1] + (unfreeze(past["cache"]),) + outputs[1:] + + return outputs + + def prepare_inputs_for_generation( + self, + decoder_input_ids, + max_length, + attention_mask: Optional[jnp.DeviceArray] = None, + decoder_attention_mask: Optional[jnp.DeviceArray] = None, + encoder_outputs=None, + **kwargs + ): + # initializing the cache + batch_size, seq_length = decoder_input_ids.shape + + past_key_values = self.init_cache(batch_size, max_length, encoder_outputs) + # Note that usually one would have to put 0's in the attention_mask for x > input_ids.shape[-1] and x < cache_length. + # But since the decoder uses a causal mask, those positions are masked anyways. + # Thus we can create a single static attention_mask here, which is more efficient for compilation + extended_attention_mask = jnp.ones((batch_size, max_length), dtype="i4") + if decoder_attention_mask is not None: + position_ids = decoder_attention_mask.cumsum(axis=-1) - 1 + extended_attention_mask = lax.dynamic_update_slice(extended_attention_mask, decoder_attention_mask, (0, 0)) + else: + position_ids = jnp.broadcast_to(jnp.arange(seq_length, dtype="i4")[None, :], (batch_size, seq_length)) + + return { + "past_key_values": past_key_values, + "encoder_outputs": encoder_outputs, + "encoder_attention_mask": attention_mask, + "decoder_attention_mask": extended_attention_mask, + "decoder_position_ids": position_ids, + } + + def update_inputs_for_generation(self, model_outputs, model_kwargs): + model_kwargs["past_key_values"] = model_outputs.past_key_values + model_kwargs["decoder_position_ids"] = model_kwargs["decoder_position_ids"][:, -1:] + 1 + return model_kwargs + + +FLAX_OPT_CONDITIONAL_GENERATION_DOCSTRING = """ + Returns: + + Summarization example: + + ```python + >>> from transformers import OPTTokenizer, FlaxOPTForConditionalGeneration + + >>> model = FlaxOPTForConditionalGeneration.from_pretrained("facebook/opt-large-cnn") + >>> tokenizer = OPTTokenizer.from_pretrained("facebook/opt-large-cnn") + + >>> ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs." + >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors="np") + + >>> # Generate Summary + >>> summary_ids = model.generate(inputs["input_ids"]).sequences + >>> print(tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)) + ``` + + Mask filling example: + + ```python + >>> import jax + >>> from transformers import OPTTokenizer, FlaxOPTForConditionalGeneration + + >>> model = FlaxOPTForConditionalGeneration.from_pretrained("facebook/opt-large") + >>> tokenizer = OPTTokenizer.from_pretrained("facebook/opt-large") + + >>> TXT = "My friends are but they eat too many carbs." + >>> input_ids = tokenizer([TXT], return_tensors="jax")["input_ids"] + + >>> logits = model(input_ids).logits + >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero()[0].item() + >>> probs = jax.nn.softmax(logits[0, masked_index], axis=0) + >>> values, predictions = jax.lax.top_k(probs, k=1) + + >>> tokenizer.decode(predictions).split() + ``` +""" + +overwrite_call_docstring( + FlaxOPTForConditionalGeneration, OPT_INPUTS_DOCSTRING + FLAX_OPT_CONDITIONAL_GENERATION_DOCSTRING +) +append_replace_return_docstrings( + FlaxOPTForConditionalGeneration, output_type=FlaxSeq2SeqLMOutput, config_class=_CONFIG_FOR_DOC +) + + +# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartForSequenceClassificationModule with Bart->OPT +class FlaxOPTForSequenceClassificationModule(nn.Module): + config: OPTConfig + dtype: jnp.dtype = jnp.float32 + num_labels: Optional[int] = None + + def setup(self): + self.model = FlaxOPTModule(config=self.config, dtype=self.dtype) + self.classification_head = FlaxOPTClassificationHead( + config=self.config, + inner_dim=self.config.d_model, + num_classes=self.num_labels if self.num_labels is not None else self.config.num_labels, + pooler_dropout=self.config.classifier_dropout, + ) + + def _get_encoder_module(self): + return self.model.encoder + + def _get_decoder_module(self): + return self.model.decoder + + def __call__( + self, + input_ids, + attention_mask, + decoder_input_ids, + decoder_attention_mask, + position_ids, + decoder_position_ids, + output_attentions: bool = False, + output_hidden_states: bool = False, + return_dict: bool = True, + deterministic: bool = True, + ): + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + decoder_input_ids=decoder_input_ids, + decoder_attention_mask=decoder_attention_mask, + position_ids=position_ids, + decoder_position_ids=decoder_position_ids, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + deterministic=deterministic, + ) + + hidden_states = outputs[0] # last hidden state + + eos_mask = jnp.where(input_ids == self.config.eos_token_id, 1, 0) + + # The first condition is necessary to overcome jax._src.errors.ConcretizationTypeError during JIT compilation + if type(eos_mask) != jax.interpreters.partial_eval.DynamicJaxprTracer: + if len(jnp.unique(eos_mask.sum(1))) > 1: + raise ValueError("All examples must have the same number of tokens.") + + if any(eos_mask.sum(1) == 0): + raise ValueError("There are missing tokens in input_ids") + + # Ensure to keep 1 only for the last token for each example + eos_mask_noised = eos_mask + jnp.arange(eos_mask.shape[1]) * 1e-6 + eos_mask = jnp.where(eos_mask_noised == eos_mask_noised.max(1).reshape(-1, 1), 1, 0) + + sentence_representation = jnp.einsum("ijk, ij -> ijk", hidden_states, eos_mask).sum(1) + logits = self.classification_head(sentence_representation, deterministic=deterministic) + + if not return_dict: + output = (logits,) + outputs[1:] + return output + + return FlaxSeq2SeqSequenceClassifierOutput( + logits=logits, + decoder_hidden_states=outputs.decoder_hidden_states, + decoder_attentions=outputs.decoder_attentions, + cross_attentions=outputs.cross_attentions, + encoder_last_hidden_state=outputs.encoder_last_hidden_state, + encoder_hidden_states=outputs.encoder_hidden_states, + encoder_attentions=outputs.encoder_attentions, + ) + + +@add_start_docstrings( + """ + OPT model with a sequence classification/head on top (a linear layer on top of the pooled output) e.g. for GLUE + tasks. + """, + OPT_START_DOCSTRING, +) +# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartForSequenceClassification with Bart->OPT +class FlaxOPTForSequenceClassification(FlaxOPTPreTrainedModel): + module_class = FlaxOPTForSequenceClassificationModule + dtype = jnp.float32 + + +append_call_sample_docstring( + FlaxOPTForSequenceClassification, + _TOKENIZER_FOR_DOC, + _CHECKPOINT_FOR_DOC, + FlaxSeq2SeqSequenceClassifierOutput, + _CONFIG_FOR_DOC, +) + + +# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartForQuestionAnsweringModule with Bart->OPT +class FlaxOPTForQuestionAnsweringModule(nn.Module): + config: OPTConfig + dtype: jnp.dtype = jnp.float32 + num_labels = 2 + + def setup(self): + self.model = FlaxOPTModule(config=self.config, dtype=self.dtype) + self.qa_outputs = nn.Dense( + self.num_labels, dtype=self.dtype, kernel_init=jax.nn.initializers.normal(self.config.init_std) + ) + + def _get_encoder_module(self): + return self.model.encoder + + def _get_decoder_module(self): + return self.model.decoder + + def __call__( + self, + input_ids, + attention_mask, + decoder_input_ids, + decoder_attention_mask, + position_ids, + decoder_position_ids, + output_attentions: bool = False, + output_hidden_states: bool = False, + return_dict: bool = True, + deterministic: bool = True, + ): + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + decoder_input_ids=decoder_input_ids, + decoder_attention_mask=decoder_attention_mask, + position_ids=position_ids, + decoder_position_ids=decoder_position_ids, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + deterministic=deterministic, + ) + + sequence_output = outputs[0] + + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = jnp.split(logits, logits.shape[-1], axis=-1) + start_logits = start_logits.squeeze(-1) + end_logits = end_logits.squeeze(-1) + + if not return_dict: + output = (start_logits, end_logits) + outputs[1:] + return output + + return FlaxSeq2SeqQuestionAnsweringModelOutput( + start_logits=start_logits, + end_logits=end_logits, + decoder_hidden_states=outputs.decoder_hidden_states, + decoder_attentions=outputs.decoder_attentions, + cross_attentions=outputs.cross_attentions, + encoder_last_hidden_state=outputs.encoder_last_hidden_state, + encoder_hidden_states=outputs.encoder_hidden_states, + encoder_attentions=outputs.encoder_attentions, + ) + + +@add_start_docstrings( + """ + OPT Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear + layer on top of the hidden-states output to compute `span start logits` and `span end logits`). + """, + OPT_START_DOCSTRING, +) +# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartForQuestionAnswering with Bart->OPT +class FlaxOPTForQuestionAnswering(FlaxOPTPreTrainedModel): + module_class = FlaxOPTForQuestionAnsweringModule + dtype = jnp.float32 + + +append_call_sample_docstring( + FlaxOPTForQuestionAnswering, + _TOKENIZER_FOR_DOC, + _CHECKPOINT_FOR_DOC, + FlaxSeq2SeqQuestionAnsweringModelOutput, + _CONFIG_FOR_DOC, +) + + +# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartDecoderPreTrainedModel with BART->OPT,Bart->OPT +class FlaxOPTDecoderPreTrainedModel(FlaxPreTrainedModel): + config_class = OPTConfig + base_model_prefix: str = "model" + module_class: nn.Module = None + + def __init__( + self, + config: OPTConfig, + input_shape: Tuple[int] = (1, 1), + seed: int = 0, + dtype: jnp.dtype = jnp.float32, + _do_init: bool = True, + **kwargs + ): + config.is_decoder = True + config.is_encoder_decoder = False + module = self.module_class(config=config, dtype=dtype, **kwargs) + super().__init__(config, module, input_shape=input_shape, seed=seed, dtype=dtype, _do_init=_do_init) + + def init_weights(self, rng: jax.random.PRNGKey, input_shape: Tuple, params: FrozenDict = None) -> FrozenDict: + # init input tensors + input_ids = jnp.zeros(input_shape, dtype="i4") + attention_mask = jnp.ones_like(input_ids) + + batch_size, sequence_length = input_ids.shape + position_ids = jnp.broadcast_to(jnp.arange(sequence_length)[None, :], (batch_size, sequence_length)) + + params_rng, dropout_rng = jax.random.split(rng) + rngs = {"params": params_rng, "dropout": dropout_rng} + encoder_hidden_states = jnp.zeros(input_shape + (self.config.d_model,)) + encoder_attention_mask = attention_mask + module_init_outputs = self.module.init( + rngs, + input_ids, + attention_mask, + position_ids, + encoder_hidden_states, + encoder_attention_mask, + return_dict=False, + ) + return module_init_outputs["params"] + + def init_cache(self, batch_size, max_length): + r""" + Args: + batch_size (`int`): + batch_size used for fast auto-regressive decoding. Defines the batch size of the initialized cache. + max_length (`int`): + maximum possible length for auto-regressive decoding. Defines the sequence length of the initialized + cache. + """ + # init input variables to retrieve cache + input_ids = jnp.ones((batch_size, max_length), dtype="i4") + attention_mask = jnp.ones_like(input_ids, dtype="i4") + position_ids = jnp.broadcast_to(jnp.arange(jnp.atleast_2d(input_ids).shape[-1]), input_ids.shape) + + init_variables = self.module.init( + jax.random.PRNGKey(0), input_ids, attention_mask, position_ids, return_dict=False, init_cache=True + ) + return unfreeze(init_variables["cache"]) + + @add_start_docstrings_to_model_forward(OPT_DECODE_INPUTS_DOCSTRING) + def __call__( + self, + input_ids: jnp.ndarray, + attention_mask: Optional[jnp.ndarray] = None, + position_ids: Optional[jnp.ndarray] = None, + encoder_hidden_states: Optional[jnp.ndarray] = None, + encoder_attention_mask: Optional[jnp.ndarray] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + train: bool = False, + params: dict = None, + past_key_values: dict = None, + dropout_rng: PRNGKey = None, + ): + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.return_dict + + if encoder_hidden_states is not None and encoder_attention_mask is None: + batch_size, sequence_length = encoder_hidden_states.shape[:2] + encoder_attention_mask = jnp.ones((batch_size, sequence_length)) + + # prepare decoder inputs + if attention_mask is None: + attention_mask = jnp.ones_like(input_ids) + if position_ids is None: + batch_size, sequence_length = input_ids.shape + position_ids = jnp.broadcast_to(jnp.arange(sequence_length)[None, :], (batch_size, sequence_length)) + + # Handle any PRNG if needed + rngs = {"dropout": dropout_rng} if dropout_rng is not None else {} + + inputs = {"params": params or self.params} + + # if past_key_values are passed then cache is already initialized a private flag init_cache has to be passed + # down to ensure cache is used. It has to be made sure that cache is marked as mutable so that it can be + # changed by FlaxOPTAttention module + if past_key_values: + inputs["cache"] = past_key_values + mutable = ["cache"] + else: + mutable = False + + outputs = self.module.apply( + inputs, + input_ids=jnp.array(input_ids, dtype="i4"), + attention_mask=jnp.array(attention_mask, dtype="i4"), + position_ids=jnp.array(position_ids, dtype="i4"), + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + deterministic=not train, + rngs=rngs, + mutable=mutable, + ) + + # add updated cache to model output + if past_key_values is not None and return_dict: + outputs, past_key_values = outputs + outputs["past_key_values"] = unfreeze(past_key_values["cache"]) + return outputs + elif past_key_values is not None and not return_dict: + outputs, past_key_values = outputs + outputs = outputs[:1] + (unfreeze(past_key_values["cache"]),) + outputs[1:] + + return outputs + + +# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartDecoderWrapper with Bart->OPT +class FlaxOPTDecoderWrapper(nn.Module): + """ + This wrapper class is a helper class to correctly load pretrained checkpoints when the causal language model is + used in combination with the [`EncoderDecoderModel`] framework. + """ + + config: OPTConfig + dtype: jnp.dtype = jnp.float32 + + def setup(self): + embed_dim = self.config.d_model + embed_tokens = nn.Embed( + self.config.vocab_size, + embed_dim, + embedding_init=jax.nn.initializers.normal(self.config.init_std), + ) + self.decoder = FlaxOPTDecoder(config=self.config, embed_tokens=embed_tokens, dtype=self.dtype) + + def __call__(self, *args, **kwargs): + return self.decoder(*args, **kwargs) + + +# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartForCausalLMModule with Bart->OPT +class FlaxOPTForCausalLMModule(nn.Module): + config: OPTConfig + dtype: jnp.dtype = jnp.float32 + + def setup(self): + self.model = FlaxOPTDecoderWrapper(config=self.config, dtype=self.dtype) + self.lm_head = nn.Dense( + self.config.vocab_size, + use_bias=False, + dtype=self.dtype, + kernel_init=jax.nn.initializers.normal(self.config.init_std), + ) + + def __call__( + self, + input_ids, + attention_mask, + position_ids, + encoder_hidden_states: Optional[jnp.ndarray] = None, + encoder_attention_mask: Optional[jnp.ndarray] = None, + init_cache: bool = False, + output_attentions: bool = False, + output_hidden_states: bool = False, + return_dict: bool = True, + deterministic: bool = True, + ): + + outputs = self.model( + input_ids, + attention_mask, + position_ids, + encoder_hidden_states, + encoder_attention_mask, + deterministic=deterministic, + init_cache=init_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + hidden_states = outputs[0] + + if self.config.tie_word_embeddings: + shared_embedding = self.model.variables["params"]["decoder"]["embed_tokens"]["embedding"] + lm_logits = self.lm_head.apply({"params": {"kernel": shared_embedding.T}}, hidden_states) + else: + lm_logits = self.lm_head(hidden_states) + + if not return_dict: + return (lm_logits,) + outputs[1:] + + return FlaxCausalLMOutputWithCrossAttentions( + logits=lm_logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + cross_attentions=outputs.cross_attentions, + ) + + +@add_start_docstrings( + """ + OPT Decoder Model with a language modeling head on top (linear layer with weights tied to the input embeddings) + e.g for autoregressive tasks. + """, + OPT_START_DOCSTRING, +) +# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartForCausalLM with Bart->OPT +class FlaxOPTForCausalLM(FlaxOPTDecoderPreTrainedModel): + module_class = FlaxOPTForCausalLMModule + + def prepare_inputs_for_generation(self, input_ids, max_length, attention_mask: Optional[jnp.DeviceArray] = None): + # initializing the cache + batch_size, seq_length = input_ids.shape + + past_key_values = self.init_cache(batch_size, max_length) + # Note that usually one would have to put 0's in the attention_mask for x > input_ids.shape[-1] and x < cache_length. + # But since the decoder uses a causal mask, those positions are masked anyway. + # Thus, we can create a single static attention_mask here, which is more efficient for compilation + extended_attention_mask = jnp.ones((batch_size, max_length), dtype="i4") + if attention_mask is not None: + position_ids = attention_mask.cumsum(axis=-1) - 1 + extended_attention_mask = lax.dynamic_update_slice(extended_attention_mask, attention_mask, (0, 0)) + else: + position_ids = jnp.broadcast_to(jnp.arange(seq_length, dtype="i4")[None, :], (batch_size, seq_length)) + + return { + "past_key_values": past_key_values, + "attention_mask": extended_attention_mask, + "position_ids": position_ids, + } + + def update_inputs_for_generation(self, model_outputs, model_kwargs): + model_kwargs["past_key_values"] = model_outputs.past_key_values + model_kwargs["position_ids"] = model_kwargs["position_ids"][:, -1:] + 1 + return model_kwargs + + +append_call_sample_docstring( + FlaxOPTForCausalLM, + _TOKENIZER_FOR_DOC, + _CHECKPOINT_FOR_DOC, + FlaxCausalLMOutputWithCrossAttentions, + _CONFIG_FOR_DOC, +) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py new file mode 100644 index 0000000000000..4548579182706 --- /dev/null +++ b/src/transformers/models/opt/modeling_opt.py @@ -0,0 +1,1818 @@ +# coding=utf-8 +# Copyright 2021 The Fairseq Authors and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch OPT model.""" +import copy +import math +import random +import warnings +from typing import List, Optional, Tuple, Union, Dict + +import torch +import torch.utils.checkpoint +import torch.nn.functional as F +from torch import Tensor +from torch import nn +from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss + +from ...activations import ACT2FN +from ...modeling_outputs import ( + BaseModelOutput, + BaseModelOutputWithPastAndCrossAttentions, + CausalLMOutputWithCrossAttentions, + Seq2SeqLMOutput, + Seq2SeqModelOutput, + Seq2SeqQuestionAnsweringModelOutput, + Seq2SeqSequenceClassifierOutput, +) +from ...modeling_utils import PreTrainedModel +from ...utils import ( + add_code_sample_docstrings, + add_end_docstrings, + add_start_docstrings, + add_start_docstrings_to_model_forward, + logging, + replace_return_docstrings, +) +from .configuration_opt import OPTConfig + + +logger = logging.get_logger(__name__) + +_CHECKPOINT_FOR_DOC = "" +_CONFIG_FOR_DOC = "OPTConfig" +_TOKENIZER_FOR_DOC = "OPTTokenizer" + +# Base model docstring +_EXPECTED_OUTPUT_SHAPE = [1, 8, 768] + +# SequenceClassification docstring +_CHECKPOINT_FOR_SEQUENCE_CLASSIFICATION = "valhalla/opt-large-sst2" +_SEQ_CLASS_EXPECTED_LOSS = 0.0 +_SEQ_CLASS_EXPECTED_OUTPUT = "'POSITIVE'" + +# QuestionAsnwering docstring +_CHECKPOINT_FOR_QA = "valhalla/opt-large-finetuned-squadv1" +_QA_EXPECTED_LOSS = 0.59 +_QA_EXPECTED_OUTPUT = "' nice puppet'" + + +OPT_PRETRAINED_MODEL_ARCHIVE_LIST = [ + "", + # See all OPT models at https://huggingface.co/models?filter=opt +] + + + +def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int): + """ + Shift input ids one token to the right. + """ + shifted_input_ids = input_ids.new_zeros(input_ids.shape) + shifted_input_ids[:, 1:] = input_ids[:, :-1].clone() + shifted_input_ids[:, 0] = decoder_start_token_id + + if pad_token_id is None: + raise ValueError("self.model.config.pad_token_id has to be defined.") + # replace possible -100 values in labels by `pad_token_id` + shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id) + + return shifted_input_ids + + +def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): + """ + Make causal mask used for bi-directional self-attention. + """ + bsz, tgt_len = input_ids_shape + mask = torch.full((tgt_len, tgt_len), float("-inf")) + mask_cond = torch.arange(mask.size(-1)) + mask.masked_fill_(mask_cond < (mask_cond + 1).view(mask.size(-1), 1), 0) + mask = mask.to(dtype) + + if past_key_values_length > 0: + mask = torch.cat([torch.zeros(tgt_len, past_key_values_length, dtype=dtype), mask], dim=-1) + return mask[None, None, :, :].expand(bsz, 1, tgt_len, tgt_len + past_key_values_length) + + +def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): + """ + Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. + """ + bsz, src_len = mask.size() + tgt_len = tgt_len if tgt_len is not None else src_len + + expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype) + + inverted_mask = 1.0 - expanded_mask + + return inverted_mask.masked_fill(inverted_mask.bool(), torch.finfo(dtype).min) + + +# Copied from transformers.models.bart.modeling_bart.BartLearnedPositionalEmbedding with Bart->OPT +# class OPTLearnedPositionalEmbedding(nn.Embedding): +# """ +# This module learns positional embeddings up to a fixed maximum size. +# """ + +# def __init__(self, num_embeddings: int, embedding_dim: int): +# # OPT is set up so that if padding_idx is specified then offset the embedding ids by 2 +# # and adjust num_embeddings appropriately. Other models don't have this hack +# self.offset = 2 +# super().__init__(num_embeddings + self.offset, embedding_dim) + +# def forward(self, input_ids_shape: torch.Size, past_key_values_length: int = 0): +# """`input_ids_shape` is expected to be [bsz x seqlen].""" +# bsz, seq_len = input_ids_shape[:2] +# positions = torch.arange( +# past_key_values_length, past_key_values_length + seq_len, dtype=torch.long, device=self.weight.device +# ) +# return super().forward(positions + self.offset) + +def make_positions(tensor, padding_idx: int, onnx_trace: bool = False): + """Replace non-padding symbols with their position numbers. + + Position numbers begin at padding_idx+1. Padding symbols are ignored. + """ + # The series of casts and type-conversions here are carefully + # balanced to both work with ONNX export and XLA. In particular XLA + # prefers ints, cumsum defaults to output longs, and ONNX doesn't know + # how to handle the dtype kwarg in cumsum. + mask = tensor.ne(padding_idx).int() + return (torch.cumsum(mask, dim=1).type_as(mask) * mask).long() + padding_idx + +class LearnedPositionalEmbedding(nn.Embedding): + """ + This module learns positional embeddings up to a fixed maximum size. + Padding ids are ignored by either offsetting based on padding_idx + or by setting padding_idx to None and ensuring that the appropriate + position ids are passed to the forward function. + """ + + def __init__(self, num_embeddings: int, embedding_dim: int, padding_idx: int=1): + super().__init__(num_embeddings, embedding_dim, padding_idx) + self.onnx_trace = False + if self.padding_idx is not None: + self.max_positions = self.num_embeddings - self.padding_idx - 1 + else: + self.max_positions = self.num_embeddings + + def forward( + self, + input: Tensor, + incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] = None, + positions: Optional[Tensor] = None, + ): + """Input is expected to be of size [bsz x seqlen].""" + assert (positions is None) or ( + self.padding_idx is None + ), "If positions is pre-computed then padding_idx should not be set." + + # we cannot use incremental state here because we must be aware of + # padding. + + if positions is None: + positions = make_positions( + input, self.padding_idx, onnx_trace=self.onnx_trace + ) + return F.embedding( + positions, + self.weight, + self.padding_idx, + self.max_norm, + self.norm_type, + self.scale_grad_by_freq, + self.sparse, + ) + +# Copied from transformers.models.bart.modeling_bart.BartAttention with Bart->OPT +class OPTAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + embed_dim: int, + num_heads: int, + dropout: float = 0.1, + is_decoder: bool = True, + bias: bool = True, + add_zero_attn = False + ): + super().__init__() + self.embed_dim = embed_dim + self.num_heads = num_heads + self.dropout = dropout + self.head_dim = embed_dim // num_heads + self.add_zero_attn = add_zero_attn + + if (self.head_dim * num_heads) != self.embed_dim: + raise ValueError( + f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}" + f" and `num_heads`: {num_heads})." + ) + self.scaling = self.head_dim**-0.5 + self.is_decoder = is_decoder + + self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + key_value_states: Optional[torch.Tensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + attention_mask: Optional[torch.Tensor] = None, + layer_head_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, + self_attn_padding_mask: Optional[Tensor] = None, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + """Input shape: Batch x Time x Channel""" + + # if key_value_states are provided this layer is used as a cross-attention layer + # for the decoder + is_cross_attention = key_value_states is not None + residual = hidden_states + + # create key padding mask + assert torch.equal(attention_mask[0, 0, :, :], attention_mask[-1, 0, :, :]) + attention_mask = attention_mask[0, 0, :, :] + + return F.multi_head_attention_forward( + hidden_states, + hidden_states, + hidden_states, + self.embed_dim, + self.num_heads, + torch.empty([0]), + torch.cat((self.q_proj.bias, self.k_proj.bias, self.v_proj.bias)), + None, + None, + self.add_zero_attn, + self.dropout, + self.out_proj.weight, + self.out_proj.bias, + self.training, + self_attn_padding_mask, + False, + attention_mask, + use_separate_proj_weight=True, + q_proj_weight=self.q_proj.weight, + k_proj_weight=self.k_proj.weight, + v_proj_weight=self.v_proj.weight, + ) + + # bsz, tgt_len, _ = hidden_states.size() + + # # get query proj + # query_states = self.q_proj(hidden_states) * self.scaling + # # get key, value proj + # if is_cross_attention and past_key_value is not None: + # # reuse k,v, cross_attentions + # key_states = past_key_value[0] + # value_states = past_key_value[1] + # elif is_cross_attention: + # # cross_attentions + # key_states = self._shape(self.k_proj(key_value_states), -1, bsz) + # value_states = self._shape(self.v_proj(key_value_states), -1, bsz) + # elif past_key_value is not None: + # # reuse k, v, self_attention + # key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + # value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + # key_states = torch.cat([past_key_value[0], key_states], dim=2) + # value_states = torch.cat([past_key_value[1], value_states], dim=2) + # else: + # # self_attention + # key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + # value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + + # if self.is_decoder: + # # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. + # # Further calls to cross_attention layer can then reuse all cross-attention + # # key/value_states (first "if" case) + # # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of + # # all previous decoder key/value_states. Further calls to uni-directional self-attention + # # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) + # # if encoder bi-directional self-attention `past_key_value` is always `None` + # past_key_value = (key_states, value_states) + + # proj_shape = (bsz * self.num_heads, -1, self.head_dim) + # query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) + # key_states = key_states.view(*proj_shape) + # value_states = value_states.view(*proj_shape) + + # src_len = key_states.size(1) + # attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + + # if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): + # raise ValueError( + # f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is {attn_weights.size()}" + # ) + + # if attention_mask is not None: + # if attention_mask.size() != (bsz, 1, tgt_len, src_len): + # raise ValueError( + # f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" + # ) + # attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask + # attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + # attn_weights = nn.functional.softmax(attn_weights, dim=-1) + + # if layer_head_mask is not None: + # if layer_head_mask.size() != (self.num_heads,): + # raise ValueError( + # f"Head mask for a single layer should be of size {(self.num_heads,)}, but is {layer_head_mask.size()}" + # ) + # attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + # attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + # if output_attentions: + # # this operation is a bit awkward, but it's required to + # # make sure that attn_weights keeps its gradient. + # # In order to do so, attn_weights have to be reshaped + # # twice and have to be reused in the following + # attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + # attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) + # else: + # attn_weights_reshaped = None + + # attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + + # attn_output = torch.bmm(attn_probs, value_states) + + # if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): + # raise ValueError( + # f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is {attn_output.size()}" + # ) + + # attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim) + # attn_output = attn_output.transpose(1, 2) + + # # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be + # # partitioned aross GPUs when using tensor-parallelism. + # attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim) + + # attn_output = self.out_proj(attn_output) + + # return attn_output + residual, attn_weights_reshaped, past_key_value + + +# Copied from transformers.models.bart.modeling_bart.BartEncoderLayer with Bart->OPT +class OPTEncoderLayer(nn.Module): + def __init__(self, config: OPTConfig): + super().__init__() + self.embed_dim = config.d_model + self.self_attn = OPTAttention( + embed_dim=self.embed_dim, + num_heads=config.encoder_attention_heads, + dropout=config.attention_dropout, + ) + self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim) + self.dropout = config.dropout + self.activation_fn = ACT2FN[config.activation_function] + self.activation_dropout = config.activation_dropout + self.fc1 = nn.Linear(self.embed_dim, config.encoder_ffn_dim) + self.fc2 = nn.Linear(config.encoder_ffn_dim, self.embed_dim) + self.final_layer_norm = nn.LayerNorm(self.embed_dim) + + def forward( + self, + hidden_states: torch.FloatTensor, + attention_mask: torch.FloatTensor, + layer_head_mask: torch.FloatTensor, + output_attentions: Optional[bool] = False, + ) -> Tuple[torch.FloatTensor, Optional[torch.FloatTensor]]: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(seq_len, batch, embed_dim)` + attention_mask (`torch.FloatTensor`): attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size + `(encoder_attention_heads,)`. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + """ + residual = hidden_states + hidden_states, attn_weights, _ = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + layer_head_mask=layer_head_mask, + output_attentions=output_attentions, + ) + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + hidden_states = residual + hidden_states + hidden_states = self.self_attn_layer_norm(hidden_states) + + residual = hidden_states + # hidden_states = self.activation_fn(self.fc1(hidden_states)) + hidden_states = nn.functional.dropout(hidden_states, p=self.activation_dropout, training=self.training) + hidden_states = self.fc2(hidden_states) + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + hidden_states = residual + hidden_states + hidden_states = self.final_layer_norm(hidden_states) + + if hidden_states.dtype == torch.float16 and ( + torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any() + ): + clamp_value = torch.finfo(hidden_states.dtype).max - 1000 + hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value) + + outputs = (hidden_states,) + + if output_attentions: + outputs += (attn_weights,) + + return outputs + + +# Copied from transformers.models.bart.modeling_bart.BartDecoderLayer with Bart->OPT +class OPTDecoderLayer(nn.Module): + def __init__(self, config: OPTConfig): + super().__init__() + self.embed_dim = config.d_model + + self.self_attn = OPTAttention( + embed_dim=self.embed_dim, + num_heads=config.decoder_attention_heads, + dropout=config.attention_dropout, + is_decoder=True, + ) + self.dropout = config.dropout + self.activation_fn = ACT2FN["relu"] + # self.activation_fn = ACT2FN[config.activation_function] + + self.activation_dropout = config.activation_dropout + + self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim) + self.fc1 = nn.Linear(self.embed_dim, config.decoder_ffn_dim) + self.fc2 = nn.Linear(config.decoder_ffn_dim, self.embed_dim) + self.final_layer_norm = nn.LayerNorm(self.embed_dim) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + layer_head_mask: Optional[torch.Tensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + output_attentions: Optional[bool] = False, + use_cache: Optional[bool] = False, + self_attn_padding_mask: Optional[Tensor] = None, + ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`): attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + encoder_hidden_states (`torch.FloatTensor`): + cross attention input to the layer of shape `(batch, seq_len, embed_dim)` + encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size + `(encoder_attention_heads,)`. + cross_attn_layer_head_mask (`torch.FloatTensor`): mask for cross-attention heads in a given layer of + size `(decoder_attention_heads,)`. + past_key_value (`Tuple(torch.FloatTensor)`): cached past key and value projection states + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + """ + + # hidden_states = hidden_states.permute(1, 0, 2) + residual = hidden_states + + + + # Self Attention + # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 + self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None + # add present self-attn cache to positions 1,2 of present_key_value tuple + hidden_states, self_attn_weights = self.self_attn( + hidden_states=hidden_states, + past_key_value=self_attn_past_key_value, + attention_mask=attention_mask, + layer_head_mask=layer_head_mask, + output_attentions=output_attentions, + self_attn_padding_mask=self_attn_padding_mask, + ) + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + hidden_states = residual + hidden_states + hidden_states = self.self_attn_layer_norm(hidden_states) + # Cross-Attention Block + cross_attn_present_key_value = None + cross_attn_weights = None + + # Fully Connected + hidden_states_shape = hidden_states.shape + hidden_states = hidden_states.reshape(-1, hidden_states.size(-1)) + residual = hidden_states + hidden_states = self.fc1(hidden_states) + hidden_states = self.activation_fn(hidden_states) + #hidden_states = nn.functional.dropout(hidden_states, p=self.activation_dropout, training=self.training) + hidden_states = self.fc2(hidden_states) + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + + hidden_states = (residual + hidden_states).view(hidden_states_shape) + # hidden_states = residual + hidden_states + hidden_states = self.final_layer_norm(hidden_states) + + outputs = (hidden_states,) + + if output_attentions: + outputs += (self_attn_weights, cross_attn_weights) + + # if use_cache: + # outputs += (present_key_value,) + + return outputs + + +# Copied from transformers.models.bart.modeling_bart.BartClassificationHead with Bart->OPT +class OPTClassificationHead(nn.Module): + """Head for sentence-level classification tasks.""" + + def __init__( + self, + input_dim: int, + inner_dim: int, + num_classes: int, + pooler_dropout: float, + ): + super().__init__() + self.dense = nn.Linear(input_dim, inner_dim) + self.dropout = nn.Dropout(p=pooler_dropout) + self.out_proj = nn.Linear(inner_dim, num_classes) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.dropout(hidden_states) + hidden_states = self.dense(hidden_states) + hidden_states = torch.tanh(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.out_proj(hidden_states) + return hidden_states + + +# Copied from transformers.models.bart.modeling_bart.BartPretrainedModel with Bart->OPT +class OPTPretrainedModel(PreTrainedModel): + config_class = OPTConfig + base_model_prefix = "model" + supports_gradient_checkpointing = True + _keys_to_ignore_on_load_unexpected = [r"encoder\.version", r"decoder\.version"] + + def _init_weights(self, module): + std = self.config.init_std + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, (OPTDecoder, OPTEncoder)): + module.gradient_checkpointing = value + + @property + def dummy_inputs(self): + pad_token = self.config.pad_token_id + input_ids = torch.tensor([[0, 6, 10, 4, 2], [0, 8, 12, 2, pad_token]], device=self.device) + dummy_inputs = { + "attention_mask": input_ids.ne(pad_token), + "input_ids": input_ids, + } + return dummy_inputs + + +# Copied from transformers.models.bart.modeling_bart.PretrainedBartModel with Bart->OPT +class PretrainedOPTModel(OPTPretrainedModel): + def __init_subclass__(self): + warnings.warn( + "The class `PretrainedOPTModel` has been depreciated, please use `OPTPretrainedModel` instead.", + FutureWarning, + ) + + +OPT_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`OPTConfig`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + +OPT_GENERATION_EXAMPLE = r""" + Summarization example: + + ```python + >>> from transformers import OPTTokenizer, OPTForConditionalGeneration + + >>> model = OPTForConditionalGeneration.from_pretrained("facebook/opt-large-cnn") + >>> tokenizer = OPTTokenizer.from_pretrained("facebook/opt-large-cnn") + + >>> ARTICLE_TO_SUMMARIZE = ( + ... "PG&E stated it scheduled the blackouts in response to forecasts for high winds " + ... "amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were " + ... "scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow." + ... ) + >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors="pt") + + >>> # Generate Summary + >>> summary_ids = model.generate(inputs["input_ids"], num_beams=2, min_length=0, max_length=20) + >>> tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + 'PG&E scheduled the blackouts in response to forecasts for high winds amid dry conditions' + ``` + + Mask filling example: + + ```python + >>> from transformers import OPTTokenizer, OPTForConditionalGeneration + + >>> tokenizer = OPTTokenizer.from_pretrained("") + >>> model = OPTForConditionalGeneration.from_pretrained("") + + >>> TXT = "My friends are but they eat too many carbs." + >>> input_ids = tokenizer([TXT], return_tensors="pt")["input_ids"] + >>> logits = model(input_ids).logits + + >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item() + >>> probs = logits[0, masked_index].softmax(dim=0) + >>> values, predictions = probs.topk(5) + + >>> tokenizer.decode(predictions).split() + ['not', 'good', 'healthy', 'great', 'very'] + ``` +""" + +OPT_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + + Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + OPT uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + For translation and summarization training, `decoder_input_ids` should be provided. If no + `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right + for denoising pre-training following the paper. + decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also + be used by default. + + If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_inputs`] and + modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information + on the default strategy. + head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, + 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): + Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) + `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of + hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape + `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape + `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. + + Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention + blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that + don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all + `decoder_input_ids` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape + `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you + can choose to directly pass an embedded representation. This is useful if you want more control over how to + convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. + decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded + representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be + input (see `past_key_values`). This is useful if you want more control over how to convert + `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. + + If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value + of `inputs_embeds`. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + + +# Copied from transformers.models.bart.modeling_bart.BartDecoder with Bart->OPT +class OPTDecoder(OPTPretrainedModel): + """ + Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`OPTDecoderLayer`] + + Args: + config: OPTConfig + embed_tokens (nn.Embedding): output embedding + """ + + def __init__(self, config: OPTConfig): + super().__init__(config) + self.dropout = config.dropout + self.layerdrop = config.decoder_layerdrop + self.padding_idx = config.pad_token_id + self.max_target_positions = config.max_position_embeddings + self.share_input_output_embed = config.share_input_output_embed + self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0 + self.cross_self_attention = False # TODO add it on the config + + self.embed_tokens = nn.Embedding(config.vocab_size, config.embed_dim, self.padding_idx) + + if self.padding_idx is not None: + num_embeddings = config.max_position_embeddings + self.padding_idx + 1 + self.embed_positions = LearnedPositionalEmbedding( + num_embeddings, + config.d_model, + self.padding_idx, + ) + + self.project_out_dim = ( + nn.Linear(config.d_model, config.embed_dim, bias=False) + if config.embed_dim != config.d_model + else None + ) + + + self.project_in_dim = ( + nn.Linear(config.embed_dim, config.d_model, bias=False) + if config.d_model != config.embed_dim + else None + ) + + + self.output_projection = nn.Linear( + config.embed_dim, config.vocab_size, bias=False + ) + + + self.layers = nn.ModuleList([OPTDecoderLayer(config) for _ in range(config.decoder_layers)]) + + self.gradient_checkpointing = False + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.embed_tokens + + def set_input_embeddings(self, value): + self.embed_tokens = value + + def _prepare_decoder_attention_mask(self, attention_mask, input_shape, inputs_embeds, past_key_values_length): + # create causal mask + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + combined_attention_mask = None + if input_shape[-1] > 1: + combined_attention_mask = _make_causal_mask( + input_shape, inputs_embeds.dtype, past_key_values_length=past_key_values_length + ).to(self.device) + + if attention_mask is not None: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + expanded_attn_mask = _expand_mask(attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]) + combined_attention_mask = ( + expanded_attn_mask if combined_attention_mask is None else expanded_attn_mask + combined_attention_mask + ) + + return combined_attention_mask + + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.Tensor] = None, + self_attn_padding_mask: Optional[torch.Tensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]: + r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you + provide it. + + Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention + of the decoder. + encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): + Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values + selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules in the decoder to avoid performing + cross-attention on hidden heads. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of + shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of + shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. + + Contains pre-computed hidden-states (key and values in the self-attention blocks and in the + cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those + that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of + all `decoder_input_ids` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of + shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing + `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more + control over how to convert `input_ids` indices into associated vectors than the model's internal + embedding lookup matrix. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # retrieve input_ids and inputs_embeds + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time") + elif input_ids is not None: + input_shape = input_ids.size() + input_ids = input_ids.view(-1, input_shape[-1]) + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds") + + # past_key_values_length + past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0 + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale + + attention_mask = self._prepare_decoder_attention_mask( + attention_mask, input_shape, inputs_embeds, past_key_values_length + ) + + if self_attn_padding_mask is None and ( + self.cross_self_attention or input_ids.eq(self.padding_idx).any() + ): + self_attn_padding_mask = input_ids.eq(self.padding_idx) + + # expand encoder attention mask + if encoder_hidden_states is not None and encoder_attention_mask is not None: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + encoder_attention_mask = _expand_mask(encoder_attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]) + + # embed positions + positions = self.embed_positions(input_ids) + + if self.project_in_dim is not None: + inputs_embeds = self.project_in_dim(inputs_embeds) + + hidden_states = inputs_embeds + positions + + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + hidden_states = hidden_states.permute(1, 0, 2) + + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + all_cross_attentions = () if (output_attentions and encoder_hidden_states is not None) else None + next_decoder_cache = () if use_cache else None + + # check if head_mask/cross_attn_head_mask has a correct number of layers specified if desired + for attn_mask, mask_name in zip([head_mask, cross_attn_head_mask], ["head_mask", "cross_attn_head_mask"]): + if attn_mask is not None: + if attn_mask.size()[0] != (len(self.layers)): + raise ValueError( + f"The `{mask_name}` should be specified for {len(self.layers)} layers, but it is for {head_mask.size()[0]}." + ) + + for idx, decoder_layer in enumerate(self.layers): + # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description) + if output_hidden_states: + all_hidden_states += (hidden_states,) + dropout_probability = random.uniform(0, 1) + if self.training and (dropout_probability < self.layerdrop): + continue + + past_key_value = past_key_values[idx] if past_key_values is not None else None + + if self.gradient_checkpointing and self.training: + + if use_cache: + logger.warning( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + ) + use_cache = False + + def create_custom_forward(module): + def custom_forward(*inputs): + # None for past_key_value + return module(*inputs, output_attentions, use_cache) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(decoder_layer), + hidden_states, + attention_mask, + encoder_hidden_states, + encoder_attention_mask, + head_mask[idx] if head_mask is not None else None, + cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None, + None, + ) + else: + + layer_outputs = decoder_layer( + hidden_states, + attention_mask=attention_mask, + layer_head_mask=(head_mask[idx] if head_mask is not None else None), + past_key_value=past_key_value, + output_attentions=output_attentions, + use_cache=use_cache, + self_attn_padding_mask=self_attn_padding_mask, + ) + hidden_states = layer_outputs[0] + + # if use_cache: + # next_decoder_cache += (layer_outputs[3 if output_attentions else 1],) + + if output_attentions: + all_self_attns += (layer_outputs[1],) + + if encoder_hidden_states is not None: + all_cross_attentions += (layer_outputs[2],) + + if self.project_out_dim is not None: + hidden_states = self.project_out_dim(hidden_states) + + # add hidden states from the last decoder layer + if output_hidden_states: + all_hidden_states += (hidden_states,) + + next_cache = next_decoder_cache if use_cache else None + if not return_dict: + return tuple( + v + for v in [hidden_states, next_cache, all_hidden_states, all_self_attns, all_cross_attentions] + if v is not None + ) + return BaseModelOutputWithPastAndCrossAttentions( + last_hidden_state=hidden_states, + past_key_values=next_cache, + hidden_states=all_hidden_states, + attentions=all_self_attns, + cross_attentions=all_cross_attentions, + ) + + +@add_start_docstrings( + "The bare OPT Model outputting raw hidden-states without any specific head on top.", + OPT_START_DOCSTRING, +) +# Copied from transformers.models.bart.modeling_bart.BartModel with BART->OPT,Bart->OPT +class OPTModel(OPTPretrainedModel): + def __init__(self, config: OPTConfig): + super().__init__(config) + + # padding_idx, vocab_size = config.pad_token_id, config.vocab_size + + self.decoder = OPTDecoder(config) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.decoder.embed_tokens + + def get_decoder(self): + return self.decoder + + @add_start_docstrings_to_model_forward(OPT_INPUTS_DOCSTRING) + @add_code_sample_docstrings( + processor_class=_TOKENIZER_FOR_DOC, + checkpoint=_CHECKPOINT_FOR_DOC, + output_type=Seq2SeqModelOutput, + config_class=_CONFIG_FOR_DOC, + expected_output=_EXPECTED_OUTPUT_SHAPE, + ) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + decoder_input_ids: Optional[torch.LongTensor] = None, + decoder_attention_mask: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.Tensor] = None, + decoder_head_mask: Optional[torch.Tensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + encoder_outputs: Optional[List[torch.FloatTensor]] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + decoder_inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, Seq2SeqModelOutput]: + + # different to other models, OPT automatically creates decoder_input_ids from + # input_ids if no decoder_input_ids are provided + if decoder_input_ids is None and decoder_inputs_embeds is None: + if input_ids is None: + raise ValueError( + "If no `decoder_input_ids` or `decoder_inputs_embeds` are " + "passed, `input_ids` cannot be `None`. Please pass either " + "`input_ids` or `decoder_input_ids` or `decoder_inputs_embeds`." + ) + + decoder_input_ids = shift_tokens_right( + input_ids, self.config.pad_token_id, self.config.decoder_start_token_id + ) + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + use_cache = use_cache if use_cache is not None else self.config.use_cache + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if encoder_outputs is None: + encoder_outputs = self.encoder( + input_ids=input_ids, + attention_mask=attention_mask, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + # If the user passed a tuple for encoder_outputs, we wrap it in a BaseModelOutput when return_dict=True + elif return_dict and not isinstance(encoder_outputs, BaseModelOutput): + encoder_outputs = BaseModelOutput( + last_hidden_state=encoder_outputs[0], + hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None, + attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None, + ) + + # decoder outputs consists of (dec_features, past_key_value, dec_hidden, dec_attn) + decoder_outputs = self.decoder( + input_ids=decoder_input_ids, + attention_mask=decoder_attention_mask, + encoder_hidden_states=encoder_outputs[0], + encoder_attention_mask=attention_mask, + head_mask=decoder_head_mask, + cross_attn_head_mask=cross_attn_head_mask, + past_key_values=past_key_values, + inputs_embeds=decoder_inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + if not return_dict: + return decoder_outputs + encoder_outputs + + return Seq2SeqModelOutput( + last_hidden_state=decoder_outputs.last_hidden_state, + past_key_values=decoder_outputs.past_key_values, + decoder_hidden_states=decoder_outputs.hidden_states, + decoder_attentions=decoder_outputs.attentions, + cross_attentions=decoder_outputs.cross_attentions, + encoder_last_hidden_state=encoder_outputs.last_hidden_state, + encoder_hidden_states=encoder_outputs.hidden_states, + encoder_attentions=encoder_outputs.attentions, + ) + + +@add_start_docstrings( + "The OPT Model with a language modeling head. Can be used for summarization.", OPT_START_DOCSTRING +) +# Copied from transformers.models.bart.modeling_bart.BartForConditionalGeneration with BART->OPT,Bart->OPT +class OPTForConditionalGeneration(OPTPretrainedModel): + base_model_prefix = "model" + _keys_to_ignore_on_load_missing = [r"final_logits_bias", r"lm_head\.weight"] + + def __init__(self, config: OPTConfig): + super().__init__(config) + self.model = OPTModel(config) + self.register_buffer("final_logits_bias", torch.zeros((1, self.model.shared.num_embeddings))) + self.lm_head = nn.Linear(config.d_model, self.model.shared.num_embeddings, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_encoder(self): + return self.model.get_encoder() + + def get_decoder(self): + return self.model.get_decoder() + + def resize_token_embeddings(self, new_num_tokens: int) -> nn.Embedding: + new_embeddings = super().resize_token_embeddings(new_num_tokens) + self._resize_final_logits_bias(new_num_tokens) + return new_embeddings + + def _resize_final_logits_bias(self, new_num_tokens: int) -> None: + old_num_tokens = self.final_logits_bias.shape[-1] + if new_num_tokens <= old_num_tokens: + new_bias = self.final_logits_bias[:, :new_num_tokens] + else: + extra_bias = torch.zeros((1, new_num_tokens - old_num_tokens), device=self.final_logits_bias.device) + new_bias = torch.cat([self.final_logits_bias, extra_bias], dim=1) + self.register_buffer("final_logits_bias", new_bias) + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + @add_start_docstrings_to_model_forward(OPT_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) + @add_end_docstrings(OPT_GENERATION_EXAMPLE) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + decoder_input_ids: Optional[torch.LongTensor] = None, + decoder_attention_mask: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.Tensor] = None, + decoder_head_mask: Optional[torch.Tensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + encoder_outputs: Optional[List[torch.FloatTensor]] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + decoder_inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, Seq2SeqLMOutput]: + r""" + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Returns: + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if labels is not None: + if use_cache: + logger.warning("The `use_cache` argument is changed to `False` since `labels` is provided.") + use_cache = False + if decoder_input_ids is None and decoder_inputs_embeds is None: + decoder_input_ids = shift_tokens_right( + labels, self.config.pad_token_id, self.config.decoder_start_token_id + ) + + outputs = self.model( + input_ids, + attention_mask=attention_mask, + decoder_input_ids=decoder_input_ids, + encoder_outputs=encoder_outputs, + decoder_attention_mask=decoder_attention_mask, + head_mask=head_mask, + decoder_head_mask=decoder_head_mask, + cross_attn_head_mask=cross_attn_head_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + decoder_inputs_embeds=decoder_inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + lm_logits = self.lm_head(outputs[0]) + self.final_logits_bias + + masked_lm_loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() + masked_lm_loss = loss_fct(lm_logits.view(-1, self.config.vocab_size), labels.view(-1)) + + if not return_dict: + output = (lm_logits,) + outputs[1:] + return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output + + return Seq2SeqLMOutput( + loss=masked_lm_loss, + logits=lm_logits, + past_key_values=outputs.past_key_values, + decoder_hidden_states=outputs.decoder_hidden_states, + decoder_attentions=outputs.decoder_attentions, + cross_attentions=outputs.cross_attentions, + encoder_last_hidden_state=outputs.encoder_last_hidden_state, + encoder_hidden_states=outputs.encoder_hidden_states, + encoder_attentions=outputs.encoder_attentions, + ) + + def prepare_inputs_for_generation( + self, + decoder_input_ids, + past=None, + attention_mask=None, + head_mask=None, + decoder_head_mask=None, + cross_attn_head_mask=None, + use_cache=None, + encoder_outputs=None, + **kwargs + ): + # cut decoder_input_ids if past is used + if past is not None: + decoder_input_ids = decoder_input_ids[:, -1:] + + return { + "input_ids": None, # encoder_outputs is defined. input_ids not needed + "encoder_outputs": encoder_outputs, + "past_key_values": past, + "decoder_input_ids": decoder_input_ids, + "attention_mask": attention_mask, + "head_mask": head_mask, + "decoder_head_mask": decoder_head_mask, + "cross_attn_head_mask": cross_attn_head_mask, + "use_cache": use_cache, # change this to avoid caching (presumably for debugging) + } + + def prepare_decoder_input_ids_from_labels(self, labels: torch.Tensor): + return shift_tokens_right(labels, self.config.pad_token_id, self.config.decoder_start_token_id) + + @staticmethod + def _reorder_cache(past, beam_idx): + reordered_past = () + for layer_past in past: + # cached cross_attention states don't have to be reordered -> they are always the same + reordered_past += ( + tuple(past_state.index_select(0, beam_idx) for past_state in layer_past[:2]) + layer_past[2:], + ) + return reordered_past + + +@add_start_docstrings( + """ + OPT model with a sequence classification/head on top (a linear layer on top of the pooled output) e.g. for GLUE + tasks. + """, + OPT_START_DOCSTRING, +) +# Copied from transformers.models.bart.modeling_bart.BartForSequenceClassification with BART->OPT,Bart->OPT +class OPTForSequenceClassification(OPTPretrainedModel): + def __init__(self, config: OPTConfig, **kwargs): + super().__init__(config, **kwargs) + self.model = OPTModel(config) + self.classification_head = OPTClassificationHead( + config.d_model, + config.d_model, + config.num_labels, + config.classifier_dropout, + ) + self.model._init_weights(self.classification_head.dense) + self.model._init_weights(self.classification_head.out_proj) + + @add_start_docstrings_to_model_forward(OPT_INPUTS_DOCSTRING) + @add_code_sample_docstrings( + processor_class=_TOKENIZER_FOR_DOC, + checkpoint=_CHECKPOINT_FOR_SEQUENCE_CLASSIFICATION, + output_type=Seq2SeqSequenceClassifierOutput, + config_class=_CONFIG_FOR_DOC, + expected_output=_SEQ_CLASS_EXPECTED_OUTPUT, + expected_loss=_SEQ_CLASS_EXPECTED_LOSS, + ) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + decoder_input_ids: Optional[torch.LongTensor] = None, + decoder_attention_mask: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.Tensor] = None, + decoder_head_mask: Optional[torch.Tensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + encoder_outputs: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + decoder_inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, Seq2SeqSequenceClassifierOutput]: + r""" + labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., + config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + if labels is not None: + use_cache = False + + if input_ids is None and inputs_embeds is not None: + raise NotImplementedError( + f"Passing input embeddings is currently not supported for {self.__class__.__name__}" + ) + + outputs = self.model( + input_ids, + attention_mask=attention_mask, + decoder_input_ids=decoder_input_ids, + decoder_attention_mask=decoder_attention_mask, + head_mask=head_mask, + decoder_head_mask=decoder_head_mask, + cross_attn_head_mask=cross_attn_head_mask, + encoder_outputs=encoder_outputs, + inputs_embeds=inputs_embeds, + decoder_inputs_embeds=decoder_inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + hidden_states = outputs[0] # last hidden state + + eos_mask = input_ids.eq(self.config.eos_token_id) + + if len(torch.unique_consecutive(eos_mask.sum(1))) > 1: + raise ValueError("All examples must have the same number of tokens.") + sentence_representation = hidden_states[eos_mask, :].view(hidden_states.size(0), -1, hidden_states.size(-1))[ + :, -1, : + ] + logits = self.classification_head(sentence_representation) + + loss = None + if labels is not None: + if self.config.problem_type is None: + if self.config.num_labels == 1: + self.config.problem_type = "regression" + elif self.config.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int): + self.config.problem_type = "single_label_classification" + else: + self.config.problem_type = "multi_label_classification" + + if self.config.problem_type == "regression": + loss_fct = MSELoss() + if self.config.num_labels == 1: + loss = loss_fct(logits.squeeze(), labels.squeeze()) + else: + loss = loss_fct(logits, labels) + elif self.config.problem_type == "single_label_classification": + loss_fct = CrossEntropyLoss() + loss = loss_fct(logits.view(-1, self.config.num_labels), labels.view(-1)) + elif self.config.problem_type == "multi_label_classification": + loss_fct = BCEWithLogitsLoss() + loss = loss_fct(logits, labels) + if not return_dict: + output = (logits,) + outputs[1:] + return ((loss,) + output) if loss is not None else output + + return Seq2SeqSequenceClassifierOutput( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + decoder_hidden_states=outputs.decoder_hidden_states, + decoder_attentions=outputs.decoder_attentions, + cross_attentions=outputs.cross_attentions, + encoder_last_hidden_state=outputs.encoder_last_hidden_state, + encoder_hidden_states=outputs.encoder_hidden_states, + encoder_attentions=outputs.encoder_attentions, + ) + + +@add_start_docstrings( + """ + OPT Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear + layer on top of the hidden-states output to compute `span start logits` and `span end logits`). + """, + OPT_START_DOCSTRING, +) +# Copied from transformers.models.bart.modeling_bart.BartForQuestionAnswering with BART->OPT,Bart->OPT +class OPTForQuestionAnswering(OPTPretrainedModel): + def __init__(self, config): + super().__init__(config) + + config.num_labels = 2 + self.num_labels = config.num_labels + + self.model = OPTModel(config) + self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels) + + self.model._init_weights(self.qa_outputs) + + @add_start_docstrings_to_model_forward(OPT_INPUTS_DOCSTRING) + @add_code_sample_docstrings( + processor_class=_TOKENIZER_FOR_DOC, + checkpoint=_CHECKPOINT_FOR_QA, + output_type=Seq2SeqQuestionAnsweringModelOutput, + config_class=_CONFIG_FOR_DOC, + expected_loss=_QA_EXPECTED_LOSS, + expected_output=_QA_EXPECTED_OUTPUT, + ) + def forward( + self, + input_ids: torch.Tensor = None, + attention_mask: Optional[torch.Tensor] = None, + decoder_input_ids: Optional[torch.LongTensor] = None, + decoder_attention_mask: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.Tensor] = None, + decoder_head_mask: Optional[torch.Tensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + encoder_outputs: Optional[List[torch.FloatTensor]] = None, + start_positions: Optional[torch.LongTensor] = None, + end_positions: Optional[torch.LongTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + decoder_inputs_embeds: Optional[torch.FloatTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, Seq2SeqQuestionAnsweringModelOutput]: + r""" + start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for position (index) of the start of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence + are not taken into account for computing the loss. + end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): + Labels for position (index) of the end of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence + are not taken into account for computing the loss. + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + if start_positions is not None and end_positions is not None: + use_cache = False + + outputs = self.model( + input_ids, + attention_mask=attention_mask, + decoder_input_ids=decoder_input_ids, + decoder_attention_mask=decoder_attention_mask, + head_mask=head_mask, + decoder_head_mask=decoder_head_mask, + cross_attn_head_mask=cross_attn_head_mask, + encoder_outputs=encoder_outputs, + inputs_embeds=inputs_embeds, + decoder_inputs_embeds=decoder_inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + sequence_output = outputs[0] + + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = logits.split(1, dim=-1) + start_logits = start_logits.squeeze(-1).contiguous() + end_logits = end_logits.squeeze(-1).contiguous() + + total_loss = None + if start_positions is not None and end_positions is not None: + # If we are on multi-GPU, split add a dimension + if len(start_positions.size()) > 1: + start_positions = start_positions.squeeze(-1) + if len(end_positions.size()) > 1: + end_positions = end_positions.squeeze(-1) + # sometimes the start/end positions are outside our model inputs, we ignore these terms + ignored_index = start_logits.size(1) + start_positions = start_positions.clamp(0, ignored_index) + end_positions = end_positions.clamp(0, ignored_index) + + loss_fct = CrossEntropyLoss(ignore_index=ignored_index) + start_loss = loss_fct(start_logits, start_positions) + end_loss = loss_fct(end_logits, end_positions) + total_loss = (start_loss + end_loss) / 2 + + if not return_dict: + output = ( + start_logits, + end_logits, + ) + outputs[1:] + return ((total_loss,) + output) if total_loss is not None else output + + return Seq2SeqQuestionAnsweringModelOutput( + loss=total_loss, + start_logits=start_logits, + end_logits=end_logits, + past_key_values=outputs.past_key_values, + decoder_hidden_states=outputs.decoder_hidden_states, + decoder_attentions=outputs.decoder_attentions, + cross_attentions=outputs.cross_attentions, + encoder_last_hidden_state=outputs.encoder_last_hidden_state, + encoder_hidden_states=outputs.encoder_hidden_states, + encoder_attentions=outputs.encoder_attentions, + ) + + +# Copied from transformers.models.bart.modeling_bart.BartDecoderWrapper with Bart->OPT +class OPTDecoderWrapper(OPTPretrainedModel): + """ + This wrapper class is a helper class to correctly load pretrained checkpoints when the causal language model is + used in combination with the [`EncoderDecoderModel`] framework. + """ + + def __init__(self, config): + super().__init__(config) + self.decoder = OPTDecoder(config) + + def forward(self, *args, **kwargs): + return self.decoder(*args, **kwargs) + + +# Copied from transformers.models.bart.modeling_bart.BartForCausalLM with Bart->OPT,facebook/bart-base-> +class OPTForCausalLM(OPTPretrainedModel): + def __init__(self, config): + config = copy.deepcopy(config) + config.is_decoder = True + config.is_encoder_decoder = False + super().__init__(config) + self.model = OPTDecoderWrapper(config) + + self.lm_head = self.model.decoder.output_projection + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.model.decoder.embed_tokens + + def set_input_embeddings(self, value): + self.model.decoder.embed_tokens = value + + def get_output_embeddings(self): + return self.lm_head + + def set_output_embeddings(self, new_embeddings): + self.lm_head = new_embeddings + + def set_decoder(self, decoder): + self.model.decoder = decoder + + def get_decoder(self): + return self.model.decoder + + @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + head_mask: Optional[torch.Tensor] = None, + cross_attn_head_mask: Optional[torch.Tensor] = None, + past_key_values: Optional[List[torch.FloatTensor]] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + labels: Optional[torch.LongTensor] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: + r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you + provide it. + + Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention + if the model is configured as a decoder. + encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used + in the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: + head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): + Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of + shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of + shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. The two additional + tensors are only required when the model is used as a decoder in a Sequence to Sequence model. + + Contains pre-computed hidden-states (key and values in the self-attention blocks and in the + cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those + that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of + all `decoder_input_ids` of shape `(batch_size, sequence_length)`. + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding + (see `past_key_values`). + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + + Returns: + + Example: + + ```python + >>> from transformers import OPTTokenizer, OPTForCausalLM + + >>> tokenizer = OPTTokenizer.from_pretrained("") + >>> model = OPTForCausalLM.from_pretrained("", add_cross_attention=False) + >>> assert model.config.is_decoder, f"{model.__class__} has to be configured as a decoder." + >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") + >>> outputs = model(**inputs) + + >>> logits = outputs.logits + >>> expected_shape = [1, inputs.input_ids.shape[-1], model.config.vocab_size] + >>> list(logits.shape) == expected_shape + True + ```""" + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn) + outputs = self.model.decoder( + input_ids=input_ids, + attention_mask=attention_mask, + head_mask=head_mask, + cross_attn_head_mask=cross_attn_head_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + logits = self.lm_head(outputs[0]) + + loss = None + if labels is not None: + loss_fct = CrossEntropyLoss() + loss = loss_fct(logits.view(-1, self.config.vocab_size), labels.view(-1)) + + if not return_dict: + output = (logits,) + outputs[1:] + return (loss,) + output if loss is not None else output + + return CausalLMOutputWithCrossAttentions( + loss=loss, + logits=logits, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + cross_attentions=outputs.cross_attentions, + ) + + def prepare_inputs_for_generation(self, input_ids, past=None, attention_mask=None, use_cache=None, **kwargs): + # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly + if attention_mask is None: + attention_mask = input_ids.new_ones(input_ids.shape) + + if past: + input_ids = input_ids[:, -1:] + # first step, decoder_cached_states are empty + return { + "input_ids": input_ids, # encoder_outputs is defined. input_ids not needed + "attention_mask": attention_mask, + "past_key_values": past, + "use_cache": use_cache, + } + + @staticmethod + def _reorder_cache(past, beam_idx): + reordered_past = () + for layer_past in past: + reordered_past += (tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),) + return reordered_past diff --git a/src/transformers/models/opt/modeling_tf_opt.py b/src/transformers/models/opt/modeling_tf_opt.py new file mode 100644 index 0000000000000..cbf694bac78bc --- /dev/null +++ b/src/transformers/models/opt/modeling_tf_opt.py @@ -0,0 +1,1402 @@ +# coding=utf-8 +# Copyright 2021 The Fairseq Authors and The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" TF 2.0 OPT model.""" + + +import random +from typing import Optional, Tuple, Union + +import numpy as np +import tensorflow as tf + +from ...activations_tf import get_tf_activation +from ...modeling_tf_outputs import ( + TFBaseModelOutput, + TFBaseModelOutputWithPastAndCrossAttentions, + TFSeq2SeqLMOutput, + TFSeq2SeqModelOutput, +) + +# Public API +from ...modeling_tf_utils import ( + DUMMY_INPUTS, + TFCausalLanguageModelingLoss, + TFModelInputType, + TFPreTrainedModel, + TFSharedEmbeddings, + TFWrappedEmbeddings, + keras_serializable, + unpack_inputs, +) +from ...tf_utils import shape_list, stable_softmax +from ...utils import ( + add_code_sample_docstrings, + add_end_docstrings, + add_start_docstrings, + add_start_docstrings_to_model_forward, + logging, + replace_return_docstrings, +) +from .configuration_opt import OPTConfig + + +logger = logging.get_logger(__name__) + +_CHECKPOINT_FOR_DOC = "facebook/bart-large" +_CONFIG_FOR_DOC = "OPTConfig" +_TOKENIZER_FOR_DOC = "OPTTokenizer" + + +LARGE_NEGATIVE = -1e8 + + +def shift_tokens_right(input_ids: tf.Tensor, pad_token_id: int, decoder_start_token_id: int): + pad_token_id = tf.cast(pad_token_id, input_ids.dtype) + decoder_start_token_id = tf.cast(decoder_start_token_id, input_ids.dtype) + start_tokens = tf.fill((shape_list(input_ids)[0], 1), decoder_start_token_id) + shifted_input_ids = tf.concat([start_tokens, input_ids[:, :-1]], -1) + # replace possible -100 values in labels by `pad_token_id` + shifted_input_ids = tf.where( + shifted_input_ids == -100, tf.fill(shape_list(shifted_input_ids), pad_token_id), shifted_input_ids + ) + + if tf.executing_eagerly(): + # "Verify that `labels` has only positive values and -100" + assert_gte0 = tf.debugging.assert_greater_equal(shifted_input_ids, tf.constant(0, dtype=input_ids.dtype)) + + # Make sure the assertion op is called by wrapping the result in an identity no-op + with tf.control_dependencies([assert_gte0]): + shifted_input_ids = tf.identity(shifted_input_ids) + + return shifted_input_ids + + +def _make_causal_mask(input_ids_shape: tf.TensorShape, past_key_values_length: int = 0): + """ + Make causal mask used for bi-directional self-attention. + """ + bsz, tgt_len = input_ids_shape + mask = tf.ones((tgt_len, tgt_len)) * LARGE_NEGATIVE + mask_cond = tf.range(shape_list(mask)[-1]) + + mask = tf.where(mask_cond < tf.reshape(mask_cond + 1, (shape_list(mask)[-1], 1)), 0.0, mask) + + if past_key_values_length > 0: + mask = tf.concat([tf.zeros((tgt_len, past_key_values_length)), mask], axis=-1) + + return tf.tile(mask[None, None, :, :], (bsz, 1, 1, 1)) + + +def _expand_mask(mask: tf.Tensor, tgt_len: Optional[int] = None, past_key_values_length: int = 0): + """ + Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. + """ + src_len = shape_list(mask)[1] + tgt_len = tgt_len if tgt_len is not None else src_len + one_cst = tf.constant(1.0) + mask = tf.cast(mask, dtype=one_cst.dtype) + expanded_mask = tf.tile(mask[:, None, None, :], (1, 1, tgt_len, 1)) + + return (one_cst - expanded_mask) * LARGE_NEGATIVE + + +# Copied from transformers.models.bart.modeling_tf_bart.TFBartLearnedPositionalEmbedding with Bart->OPT +class TFOPTLearnedPositionalEmbedding(TFSharedEmbeddings): + """ + This module learns positional embeddings up to a fixed maximum size. + """ + + def __init__(self, num_embeddings: int, embedding_dim: int, **kwargs): + # OPT is set up so that if padding_idx is specified then offset the embedding ids by 2 + # and adjust num_embeddings appropriately. Other models don't have this hack + self.offset = 2 + super().__init__(num_embeddings + self.offset, embedding_dim, **kwargs) + + def call(self, input_shape: tf.TensorShape, past_key_values_length: int = 0): + """Input is expected to be of size [bsz x seqlen].""" + bsz, seq_len = input_shape[:2] + + positions = tf.range(past_key_values_length, seq_len + past_key_values_length, delta=1, name="range") + return super().call(positions + self.offset) + + +# Copied from transformers.models.bart.modeling_tf_bart.TFBartAttention with Bart->OPT +class TFOPTAttention(tf.keras.layers.Layer): + """Multi-headed attention from "Attention Is All You Need""" + + def __init__( + self, + embed_dim: int, + num_heads: int, + dropout: float = 0.0, + is_decoder: bool = False, + bias: bool = True, + **kwargs, + ): + super().__init__(**kwargs) + self.embed_dim = embed_dim + + self.num_heads = num_heads + self.dropout = tf.keras.layers.Dropout(dropout) + self.head_dim = embed_dim // num_heads + if (self.head_dim * num_heads) != self.embed_dim: + raise ValueError( + f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}" + f" and `num_heads`: {num_heads})." + ) + self.scaling = self.head_dim**-0.5 + self.is_decoder = is_decoder + + self.k_proj = tf.keras.layers.Dense(embed_dim, use_bias=bias, name="k_proj") + self.q_proj = tf.keras.layers.Dense(embed_dim, use_bias=bias, name="q_proj") + self.v_proj = tf.keras.layers.Dense(embed_dim, use_bias=bias, name="v_proj") + self.out_proj = tf.keras.layers.Dense(embed_dim, use_bias=bias, name="out_proj") + + def _shape(self, tensor: tf.Tensor, seq_len: int, bsz: int): + return tf.transpose(tf.reshape(tensor, (bsz, seq_len, self.num_heads, self.head_dim)), (0, 2, 1, 3)) + + def call( + self, + hidden_states: tf.Tensor, + key_value_states: Optional[tf.Tensor] = None, + past_key_value: Optional[Tuple[Tuple[tf.Tensor]]] = None, + attention_mask: Optional[tf.Tensor] = None, + layer_head_mask: Optional[tf.Tensor] = None, + training: Optional[bool] = False, + ) -> Tuple[tf.Tensor, Optional[tf.Tensor]]: + """Input shape: Batch x Time x Channel""" + + # if key_value_states are provided this layer is used as a cross-attention layer + # for the decoder + is_cross_attention = key_value_states is not None + bsz, tgt_len, embed_dim = shape_list(hidden_states) + + # get query proj + query_states = self.q_proj(hidden_states) * self.scaling + # get key, value proj + if is_cross_attention and past_key_value is not None: + # reuse k,v, cross_attentions + key_states = past_key_value[0] + value_states = past_key_value[1] + elif is_cross_attention: + # cross_attentions + key_states = self._shape(self.k_proj(key_value_states), -1, bsz) + value_states = self._shape(self.v_proj(key_value_states), -1, bsz) + elif past_key_value is not None: + # reuse k, v, self_attention + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + key_states = tf.concat([past_key_value[0], key_states], axis=2) + value_states = tf.concat([past_key_value[1], value_states], axis=2) + else: + # self_attention + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + + if self.is_decoder: + # if cross_attention save Tuple(tf.Tensor, tf.Tensor) of all cross attention key/value_states. + # Further calls to cross_attention layer can then reuse all cross-attention + # key/value_states (first "if" case) + # if uni-directional self-attention (decoder) save Tuple(tf.Tensor, tf.Tensor) of + # all previous decoder key/value_states. Further calls to uni-directional self-attention + # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) + # if encoder bi-directional self-attention `past_key_value` is always `None` + past_key_value = (key_states, value_states) + + proj_shape = (bsz * self.num_heads, -1, self.head_dim) + query_states = tf.reshape(self._shape(query_states, tgt_len, bsz), proj_shape) + key_states = tf.reshape(key_states, proj_shape) + value_states = tf.reshape(value_states, proj_shape) + + src_len = shape_list(key_states)[1] + attn_weights = tf.matmul(query_states, key_states, transpose_b=True) + + # The tf.debugging asserts are not compliant with XLA then they + # have to be disabled in other modes than eager. + if tf.executing_eagerly(): + tf.debugging.assert_equal( + shape_list(attn_weights), + [bsz * self.num_heads, tgt_len, src_len], + message=f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is {shape_list(attn_weights)}", + ) + + if attention_mask is not None: + # The tf.debugging asserts are not compliant with XLA then they + # have to be disabled in other modes than eager. + if tf.executing_eagerly(): + tf.debugging.assert_equal( + shape_list(attention_mask), + [bsz, 1, tgt_len, src_len], + message=f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {shape_list(attention_mask)}", + ) + + attention_mask = tf.cast(attention_mask, dtype=attn_weights.dtype) + attn_weights = tf.reshape(attn_weights, (bsz, self.num_heads, tgt_len, src_len)) + attention_mask + attn_weights = tf.reshape(attn_weights, (bsz * self.num_heads, tgt_len, src_len)) + + attn_weights = stable_softmax(attn_weights, axis=-1) + + if layer_head_mask is not None: + # The tf.debugging asserts are not compliant with XLA then they + # have to be disabled in other modes than eager. + if tf.executing_eagerly(): + tf.debugging.assert_equal( + shape_list(layer_head_mask), + [self.num_heads], + message=f"Head mask for a single layer should be of size {(self.num_heads)}, but is {shape_list(layer_head_mask)}", + ) + + attn_weights = tf.reshape(layer_head_mask, (1, -1, 1, 1)) * tf.reshape( + attn_weights, (bsz, self.num_heads, tgt_len, src_len) + ) + attn_weights = tf.reshape(attn_weights, (bsz * self.num_heads, tgt_len, src_len)) + + attn_probs = self.dropout(attn_weights, training=training) + attn_output = tf.matmul(attn_probs, value_states) + + # The tf.debugging asserts are not compliant with XLA then they + # have to be disabled in other modes than eager. + if tf.executing_eagerly(): + tf.debugging.assert_equal( + shape_list(attn_output), + [bsz * self.num_heads, tgt_len, self.head_dim], + message=f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is {shape_list(attn_output)}", + ) + + attn_output = tf.transpose( + tf.reshape(attn_output, (bsz, self.num_heads, tgt_len, self.head_dim)), (0, 2, 1, 3) + ) + attn_output = tf.reshape(attn_output, (bsz, tgt_len, embed_dim)) + + attn_output = self.out_proj(attn_output) + attn_weights: tf.Tensor = tf.reshape(attn_weights, (bsz, self.num_heads, tgt_len, src_len)) + + return attn_output, attn_weights, past_key_value + + +# Copied from transformers.models.bart.modeling_tf_bart.TFBartEncoderLayer with Bart->OPT +class TFOPTEncoderLayer(tf.keras.layers.Layer): + def __init__(self, config: OPTConfig, **kwargs): + super().__init__(**kwargs) + self.embed_dim = config.d_model + self.self_attn = TFOPTAttention( + self.embed_dim, config.encoder_attention_heads, dropout=config.attention_dropout, name="self_attn" + ) + self.self_attn_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="self_attn_layer_norm") + self.dropout = tf.keras.layers.Dropout(config.dropout) + self.activation_fn = get_tf_activation(config.activation_function) + self.activation_dropout = tf.keras.layers.Dropout(config.activation_dropout) + self.fc1 = tf.keras.layers.Dense(config.encoder_ffn_dim, name="fc1") + self.fc2 = tf.keras.layers.Dense(self.embed_dim, name="fc2") + self.final_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="final_layer_norm") + + def call( + self, + hidden_states: tf.Tensor, + attention_mask: Optional[Union[np.ndarray, tf.Tensor]], + layer_head_mask: Optional[tf.Tensor], + training: Optional[bool] = False, + ) -> tf.Tensor: + """ + Args: + hidden_states (`tf.Tensor`): input to the layer of shape `(seq_len, batch, embed_dim)` + attention_mask (`tf.Tensor`): attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + layer_head_mask (`tf.Tensor`): mask for attention heads in a given layer of size + `(encoder_attention_heads,)` + """ + residual = hidden_states + hidden_states, self_attn_weights, _ = self.self_attn( + hidden_states=hidden_states, attention_mask=attention_mask, layer_head_mask=layer_head_mask + ) + + # The tf.debugging asserts are not compliant with XLA then they + # have to be disabled in other modes than eager. + if tf.executing_eagerly(): + tf.debugging.assert_equal( + shape_list(hidden_states), + shape_list(residual), + message=f"Self attn modified the shape of query {shape_list(residual)} to {shape_list(hidden_states)}", + ) + + hidden_states = self.dropout(hidden_states, training=training) + hidden_states = residual + hidden_states + hidden_states = self.self_attn_layer_norm(hidden_states) + + residual = hidden_states + hidden_states = self.activation_fn(self.fc1(hidden_states)) + hidden_states = self.activation_dropout(hidden_states, training=training) + hidden_states = self.fc2(hidden_states) + hidden_states = self.dropout(hidden_states, training=training) + hidden_states = residual + hidden_states + hidden_states = self.final_layer_norm(hidden_states) + + return hidden_states, self_attn_weights + + +# Copied from transformers.models.bart.modeling_tf_bart.TFBartDecoderLayer with Bart->OPT +class TFOPTDecoderLayer(tf.keras.layers.Layer): + def __init__(self, config: OPTConfig, **kwargs): + super().__init__(**kwargs) + self.embed_dim = config.d_model + self.self_attn = TFOPTAttention( + embed_dim=self.embed_dim, + num_heads=config.decoder_attention_heads, + dropout=config.attention_dropout, + name="self_attn", + is_decoder=True, + ) + self.dropout = tf.keras.layers.Dropout(config.dropout) + self.activation_fn = get_tf_activation(config.activation_function) + self.activation_dropout = tf.keras.layers.Dropout(config.activation_dropout) + + self.self_attn_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="self_attn_layer_norm") + self.encoder_attn = TFOPTAttention( + self.embed_dim, + config.decoder_attention_heads, + dropout=config.attention_dropout, + name="encoder_attn", + is_decoder=True, + ) + self.encoder_attn_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="encoder_attn_layer_norm") + self.fc1 = tf.keras.layers.Dense(config.decoder_ffn_dim, name="fc1") + self.fc2 = tf.keras.layers.Dense(self.embed_dim, name="fc2") + self.final_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="final_layer_norm") + + def call( + self, + hidden_states: tf.Tensor, + attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + encoder_hidden_states: Optional[Union[np.ndarray, tf.Tensor]] = None, + encoder_attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + layer_head_mask: Optional[tf.Tensor] = None, + cross_attn_layer_head_mask: Optional[tf.Tensor] = None, + past_key_value: Optional[Tuple[Tuple[Union[np.ndarray, tf.Tensor]]]] = None, + training: Optional[bool] = False, + ) -> Tuple[tf.Tensor, tf.Tensor, Tuple[Tuple[tf.Tensor]]]: + """ + Args: + hidden_states (`tf.Tensor`): input to the layer of shape `(seq_len, batch, embed_dim)` + attention_mask (`tf.Tensor`): attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + encoder_hidden_states (`tf.Tensor`): + cross attention input to the layer of shape `(seq_len, batch, embed_dim)` + encoder_attention_mask (`tf.Tensor`): encoder attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + layer_head_mask (`tf.Tensor`): mask for attention heads in a given layer of size + `(decoder_attention_heads,)` + cross_attn_layer_head_mask (`tf.Tensor`): mask for heads of the cross-attention module. + `(decoder_attention_heads,)` + past_key_value (`Tuple(tf.Tensor)`): cached past key and value projection states + """ + residual = hidden_states + + # Self Attention + # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 + self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None + # add present self-attn cache to positions 1,2 of present_key_value tuple + hidden_states, self_attn_weights, present_key_value = self.self_attn( + hidden_states=hidden_states, + past_key_value=self_attn_past_key_value, + attention_mask=attention_mask, + layer_head_mask=layer_head_mask, + ) + hidden_states = self.dropout(hidden_states, training=training) + hidden_states = residual + hidden_states + hidden_states = self.self_attn_layer_norm(hidden_states) + + # Cross-Attention Block + cross_attn_present_key_value = None + cross_attn_weights = None + if encoder_hidden_states is not None: + residual = hidden_states + + # cross_attn cached key/values tuple is at positions 3,4 of present_key_value tuple + cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None + hidden_states, cross_attn_weights, cross_attn_present_key_value = self.encoder_attn( + hidden_states=hidden_states, + key_value_states=encoder_hidden_states, + attention_mask=encoder_attention_mask, + layer_head_mask=cross_attn_layer_head_mask, + past_key_value=cross_attn_past_key_value, + ) + hidden_states = self.dropout(hidden_states, training=training) + hidden_states = residual + hidden_states + hidden_states = self.encoder_attn_layer_norm(hidden_states) + + # add cross-attn to positions 3,4 of present_key_value tuple + present_key_value = present_key_value + cross_attn_present_key_value + + # Fully Connected + residual = hidden_states + hidden_states = self.activation_fn(self.fc1(hidden_states)) + hidden_states = self.activation_dropout(hidden_states, training=training) + hidden_states = self.fc2(hidden_states) + hidden_states = self.dropout(hidden_states, training=training) + hidden_states = residual + hidden_states + hidden_states = self.final_layer_norm(hidden_states) + + return ( + hidden_states, + self_attn_weights, + cross_attn_weights, + present_key_value, + ) + + +# Copied from transformers.models.bart.modeling_tf_bart.TFBartPretrainedModel with Bart->OPT +class TFOPTPretrainedModel(TFPreTrainedModel): + config_class = OPTConfig + base_model_prefix = "model" + + @property + def dummy_inputs(self): + pad_token = 1 + input_ids = tf.cast(tf.convert_to_tensor(DUMMY_INPUTS), tf.int32) + decoder_input_ids = tf.cast(tf.convert_to_tensor(DUMMY_INPUTS), tf.int32) + dummy_inputs = { + "decoder_input_ids": decoder_input_ids, + "attention_mask": tf.math.not_equal(input_ids, pad_token), + "input_ids": input_ids, + } + return dummy_inputs + + @tf.function( + input_signature=[ + { + "input_ids": tf.TensorSpec((None, None), tf.int32, name="input_ids"), + "attention_mask": tf.TensorSpec((None, None), tf.int32, name="attention_mask"), + "decoder_input_ids": tf.TensorSpec((None, None), tf.int32, name="decoder_input_ids"), + "decoder_attention_mask": tf.TensorSpec((None, None), tf.int32, name="decoder_attention_mask"), + } + ] + ) + def serving(self, inputs): + output = self.call(inputs) + + return self.serving_output(output) + + +OPT_START_DOCSTRING = r""" + This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it + as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and + behavior. + + + + TF 2.0 models accepts two formats as inputs: + + - having all inputs as keyword arguments (like PyTorch models), or + - having all inputs as a list, tuple or dict in the first positional arguments. + + This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the + tensors in the first argument of the model call function: `model(inputs)`. + + If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the + first positional argument : + + - a single Tensor with `input_ids` only and nothing else: `model(input_ids)` + - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: + `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` + - a dictionary with one or several input Tensors associated to the input names given in the docstring: + `model({"input_ids": input_ids, "token_type_ids": token_type_ids})` + + + + Args: + config ([`OPTConfig`]): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights. +""" + + +OPT_GENERATION_EXAMPLE = r""" + Summarization example: + + ```python + >>> from transformers import OPTTokenizer, TFOPTForConditionalGeneration + + >>> model = TFOPTForConditionalGeneration.from_pretrained("facebook/opt-large") + >>> tokenizer = OPTTokenizer.from_pretrained("facebook/opt-large") + + >>> ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs." + >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors="tf") + + >>> # Generate Summary + >>> summary_ids = model.generate(inputs["input_ids"], num_beams=4, max_length=5) + >>> print(tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)) + ``` + + Mask filling example: + + ```python + >>> from transformers import OPTTokenizer, TFOPTForConditionalGeneration + + >>> tokenizer = OPTTokenizer.from_pretrained("facebook/opt-large") + >>> TXT = "My friends are but they eat too many carbs." + + >>> model = TFOPTForConditionalGeneration.from_pretrained("facebook/opt-large") + >>> input_ids = tokenizer([TXT], return_tensors="tf")["input_ids"] + >>> logits = model(input_ids).logits + >>> probs = tf.nn.softmax(logits[0]) + >>> # probs[5] is associated with the mask token + ``` +""" + + +OPT_INPUTS_DOCSTRING = r""" + Args: + input_ids (`tf.Tensor` of shape `({0})`): + Indices of input sequence tokens in the vocabulary. + + Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`tf.Tensor` of shape `({0})`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + decoder_input_ids (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*): + Indices of decoder input sequence tokens in the vocabulary. + + Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are decoder input IDs?](../glossary#decoder-input-ids) + + OPT uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` + is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + + For translation and summarization training, `decoder_input_ids` should be provided. If no + `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right + for denoising pre-training following the paper. + decoder_attention_mask (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*): + will be made by default and ignore pad tokens. It is not recommended to set this for most use cases. + head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + decoder_head_mask (`tf.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + cross_attn_head_mask (`tf.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + encoder_outputs (`tf.FloatTensor`, *optional*): + hidden states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. + of shape `(batch_size, sequence_length, hidden_size)` is a sequence of + past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers`) + contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that + don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all + `decoder_input_ids` of shape `(batch_size, sequence_length)`. + use_cache (`bool`, *optional*, defaults to `True`): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). Set to `False` during training, `True` during generation + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the + config will be used instead. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. This argument can be used only in eager mode, in graph mode the value in the config will be + used instead. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. This argument can be used in + eager mode, in graph mode the value will always be set to True. + training (`bool`, *optional*, defaults to `False`): + Whether or not to use the model in training mode (some modules like dropout modules have different + behaviors between training and evaluation). +""" + + +@keras_serializable +# Copied from transformers.models.bart.modeling_tf_bart.TFBartEncoder with Bart->OPT +class TFOPTEncoder(tf.keras.layers.Layer): + config_class = OPTConfig + """ + Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a + [`TFOPTEncoderLayer`]. + + Args: + config: OPTConfig + """ + + def __init__(self, config: OPTConfig, embed_tokens: Optional[TFSharedEmbeddings] = None, **kwargs): + super().__init__(**kwargs) + self.config = config + self.dropout = tf.keras.layers.Dropout(config.dropout) + self.layerdrop = config.encoder_layerdrop + self.padding_idx = config.pad_token_id + self.max_source_positions = config.max_position_embeddings + self.embed_scale = tf.math.sqrt(float(config.d_model)) if config.scale_embedding else 1.0 + + self.embed_tokens = embed_tokens + self.embed_positions = TFOPTLearnedPositionalEmbedding( + config.max_position_embeddings, + config.d_model, + name="embed_positions", + ) + self.layers = [TFOPTEncoderLayer(config, name=f"layers.{i}") for i in range(config.encoder_layers)] + self.layernorm_embedding = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="layernorm_embedding") + + def get_embed_tokens(self): + return self.embed_tokens + + def set_embed_tokens(self, embed_tokens): + self.embed_tokens = embed_tokens + + @unpack_inputs + def call( + self, + input_ids: Optional[TFModelInputType] = None, + inputs_embeds: Optional[Union[np.ndarray, tf.Tensor]] = None, + attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + training: Optional[bool] = False, + ) -> Union[TFBaseModelOutput, Tuple[tf.Tensor]]: + """ + Args: + input_ids (`tf.Tensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you + provide it. + + Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, `optional): + Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + """ + + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = shape_list(input_ids) + elif inputs_embeds is not None: + input_shape = shape_list(inputs_embeds)[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale + + embed_pos = self.embed_positions(input_shape) + hidden_states = inputs_embeds + embed_pos + hidden_states = self.layernorm_embedding(hidden_states) + hidden_states = self.dropout(hidden_states, training=training) + + # check attention mask and invert + if attention_mask is not None: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + attention_mask = _expand_mask(attention_mask) + else: + attention_mask = None + + encoder_states = () if output_hidden_states else None + all_attentions = () if output_attentions else None + + # check if head_mask has a correct number of layers specified if desired + # The tf.debugging asserts are not compliant with XLA then they + # have to be disabled in other modes than eager. + if head_mask is not None and tf.executing_eagerly(): + tf.debugging.assert_equal( + shape_list(head_mask)[0], + len(self.layers), + message=f"The head_mask should be specified for {len(self.layers)} layers, but it is for {shape_list(head_mask)[0]}.", + ) + + # encoder layers + for idx, encoder_layer in enumerate(self.layers): + + if output_hidden_states: + encoder_states = encoder_states + (hidden_states,) + # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description) + dropout_probability = random.uniform(0, 1) + if training and (dropout_probability < self.layerdrop): # skip the layer + continue + + hidden_states, attn = encoder_layer( + hidden_states, + attention_mask, + head_mask[idx] if head_mask is not None else None, + ) + + if output_attentions: + all_attentions += (attn,) + + if output_hidden_states: + encoder_states = encoder_states + (hidden_states,) + + if not return_dict: + return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None) + return TFBaseModelOutput( + last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions + ) + + +@keras_serializable +# Copied from transformers.models.bart.modeling_tf_bart.TFBartDecoder with Bart->OPT +class TFOPTDecoder(tf.keras.layers.Layer): + config_class = OPTConfig + """ + Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`TFOPTDecoderLayer`] + + Args: + config: OPTConfig + embed_tokens: output embedding + """ + + def __init__(self, config: OPTConfig, embed_tokens: Optional[TFSharedEmbeddings] = None, **kwargs): + super().__init__(**kwargs) + self.config = config + self.padding_idx = config.pad_token_id + self.embed_tokens = embed_tokens + self.layerdrop = config.decoder_layerdrop + self.embed_positions = TFOPTLearnedPositionalEmbedding( + config.max_position_embeddings, + config.d_model, + name="embed_positions", + ) + self.embed_scale = tf.math.sqrt(float(config.d_model)) if config.scale_embedding else 1.0 + self.layers = [TFOPTDecoderLayer(config, name=f"layers.{i}") for i in range(config.decoder_layers)] + self.layernorm_embedding = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="layernorm_embedding") + + self.dropout = tf.keras.layers.Dropout(config.dropout) + + def get_embed_tokens(self): + return self.embed_tokens + + def set_embed_tokens(self, embed_tokens): + self.embed_tokens = embed_tokens + + @unpack_inputs + def call( + self, + input_ids: Optional[TFModelInputType] = None, + inputs_embeds: Optional[Union[np.ndarray, tf.Tensor]] = None, + attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + encoder_hidden_states: Optional[Union[np.ndarray, tf.Tensor]] = None, + encoder_attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + cross_attn_head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + past_key_values: Optional[Tuple[Tuple[Union[np.ndarray, tf.Tensor]]]] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + training: Optional[bool] = False, + ) -> Union[TFBaseModelOutputWithPastAndCrossAttentions, Tuple[tf.Tensor]]: + r""" + Args: + input_ids (`tf.Tensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you + provide it. + + Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + encoder_hidden_states (`tf.Tensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention + of the decoder. + encoder_attention_mask (`tf.Tensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): + Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values + selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + head_mask (`tf.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + cross_attn_head_mask (`tf.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers` with each tuple having 2 tuples each of which has 2 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + Contains precomputed key and value hidden-states of the attention blocks. Can be used to speed up + decoding. + + If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those + that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of + all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`tf.Tensor` of + shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing + `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more + control over how to convert `input_ids` indices into associated vectors than the model's internal + embedding lookup matrix. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + """ + + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time") + elif input_ids is not None: + input_shape = shape_list(input_ids) + elif inputs_embeds is not None: + input_shape = shape_list(inputs_embeds)[:-1] + else: + raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds") + + past_key_values_length = shape_list(past_key_values[0][0])[2] if past_key_values is not None else 0 + + # embed positions + positions = self.embed_positions(input_shape, past_key_values_length) + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale + + hidden_states = inputs_embeds + + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + if input_shape[-1] > 1: + combined_attention_mask = _make_causal_mask(input_shape, past_key_values_length=past_key_values_length) + else: + combined_attention_mask = _expand_mask( + tf.ones((input_shape[0], input_shape[1] + past_key_values_length)), tgt_len=input_shape[-1] + ) + + if attention_mask is not None: + combined_attention_mask = combined_attention_mask + _expand_mask(attention_mask, tgt_len=input_shape[-1]) + + if encoder_hidden_states is not None and encoder_attention_mask is not None: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + encoder_attention_mask = _expand_mask(encoder_attention_mask, tgt_len=input_shape[-1]) + + hidden_states = self.layernorm_embedding(hidden_states + positions) + hidden_states = self.dropout(hidden_states, training=training) + + # decoder layers + all_hidden_states = () if output_hidden_states else None + all_self_attns = () if output_attentions else None + all_cross_attns = () if (output_attentions and encoder_hidden_states is not None) else None + present_key_values = () if use_cache else None + + # check if head_mask and cross_attn_head_mask have a correct number of layers specified if desired + # The tf.debugging asserts are not compliant with XLA then they + # have to be disabled in other modes than eager. + for attn_mask_name, attn_mask in [("head_mask", head_mask), ("cross_attn_head_mask", cross_attn_head_mask)]: + if attn_mask is not None and tf.executing_eagerly(): + tf.debugging.assert_equal( + shape_list(attn_mask)[0], + len(self.layers), + message=f"The {attn_mask_name} should be specified for {len(self.layers)} layers, but it is for {shape_list(attn_mask)[0]}.", + ) + + for idx, decoder_layer in enumerate(self.layers): + # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description) + if output_hidden_states: + all_hidden_states += (hidden_states,) + + dropout_probability = random.uniform(0, 1) + + if training and (dropout_probability < self.layerdrop): + continue + + past_key_value = past_key_values[idx] if past_key_values is not None else None + + hidden_states, layer_self_attn, layer_cross_attn, present_key_value = decoder_layer( + hidden_states, + attention_mask=combined_attention_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + layer_head_mask=head_mask[idx] if head_mask is not None else None, + cross_attn_layer_head_mask=cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None, + past_key_value=past_key_value, + ) + + if use_cache: + present_key_values += (present_key_value,) + + if output_attentions: + all_self_attns += (layer_self_attn,) + + if encoder_hidden_states is not None: + all_cross_attns += (layer_cross_attn,) + + if output_hidden_states: + all_hidden_states += (hidden_states,) + + if not return_dict: + return hidden_states, present_key_values, all_hidden_states, all_self_attns, all_cross_attns + else: + return TFBaseModelOutputWithPastAndCrossAttentions( + last_hidden_state=hidden_states, + past_key_values=present_key_values, + hidden_states=all_hidden_states, + attentions=all_self_attns, + cross_attentions=all_cross_attns, + ) + + +@keras_serializable +# Copied from transformers.models.bart.modeling_tf_bart.TFBartMainLayer with Bart->OPT +class TFOPTMainLayer(tf.keras.layers.Layer): + config_class = OPTConfig + + def __init__(self, config: OPTConfig, load_weight_prefix=None, **kwargs): + super().__init__(**kwargs) + self.config = config + self.shared = TFSharedEmbeddings(config.vocab_size, config.d_model, config.pad_token_id, name="model.shared") + + # set tf scope correctly + if load_weight_prefix is None: + load_weight_prefix = "model.shared" + + with tf.compat.v1.variable_scope(load_weight_prefix) as shared_abs_scope_name: + pass + + # Wraps layer to avoid problems with weight restoring and ensuring we're in the correct TF scope. + embed_tokens = TFWrappedEmbeddings(self.shared, abs_scope_name=shared_abs_scope_name) + embed_tokens.vocab_size = self.shared.vocab_size + embed_tokens.hidden_size = self.shared.hidden_size + + self.encoder = TFOPTEncoder(config, embed_tokens, name="encoder") + self.decoder = TFOPTDecoder(config, embed_tokens, name="decoder") + + def get_input_embeddings(self): + return self.shared + + def set_input_embeddings(self, new_embeddings): + self.shared.weight = new_embeddings + self.shared.vocab_size = self.shared.weight.shape[0] + # retrieve correct absolute scope for embed token wrapper + with tf.compat.v1.variable_scope("model.shared") as shared_abs_scope_name: + pass + # Wraps layer to avoid problems with weight restoring and ensuring we're in the correct TF scope. + embed_tokens = TFWrappedEmbeddings(self.shared, abs_scope_name=shared_abs_scope_name) + self.encoder.set_embed_tokens(embed_tokens) + self.decoder.set_embed_tokens(embed_tokens) + + @unpack_inputs + def call( + self, + input_ids: Optional[TFModelInputType] = None, + attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + decoder_input_ids: Optional[Union[np.ndarray, tf.Tensor]] = None, + decoder_attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + decoder_head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + cross_attn_head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + encoder_outputs: Optional[Union[Tuple, TFBaseModelOutput]] = None, + past_key_values: Optional[Tuple[Tuple[Union[np.ndarray, tf.Tensor]]]] = None, + inputs_embeds: Optional[Union[np.ndarray, tf.Tensor]] = None, + decoder_inputs_embeds: Optional[Union[np.ndarray, tf.Tensor]] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + training: Optional[bool] = False, + **kwargs + ) -> Union[TFSeq2SeqModelOutput, Tuple[tf.Tensor]]: + + if decoder_input_ids is None and decoder_inputs_embeds is None: + use_cache = False + + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + + if decoder_input_ids is None and input_ids is not None: + decoder_input_ids = shift_tokens_right( + input_ids, self.config.pad_token_id, self.config.decoder_start_token_id + ) + + if encoder_outputs is None: + encoder_outputs = self.encoder( + input_ids=input_ids, + attention_mask=attention_mask, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + training=training, + ) + # If the user passed a tuple for encoder_outputs, we wrap it in a TFBaseModelOutput when return_dict=True + elif return_dict and not isinstance(encoder_outputs, TFBaseModelOutput): + encoder_outputs = TFBaseModelOutput( + last_hidden_state=encoder_outputs[0], + hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None, + attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None, + ) + # If the user passed a TFBaseModelOutput for encoder_outputs, we wrap it in a tuple when return_dict=False + elif not return_dict and not isinstance(encoder_outputs, tuple): + encoder_outputs = encoder_outputs.to_tuple() + + decoder_outputs = self.decoder( + decoder_input_ids, + attention_mask=decoder_attention_mask, + encoder_hidden_states=encoder_outputs[0], + encoder_attention_mask=attention_mask, + head_mask=decoder_head_mask, + cross_attn_head_mask=cross_attn_head_mask, + past_key_values=past_key_values, + inputs_embeds=decoder_inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + training=training, + ) + + if not return_dict: + return decoder_outputs + encoder_outputs + + return TFSeq2SeqModelOutput( + last_hidden_state=decoder_outputs.last_hidden_state, + past_key_values=decoder_outputs.past_key_values, + decoder_hidden_states=decoder_outputs.hidden_states, + decoder_attentions=decoder_outputs.attentions, + cross_attentions=decoder_outputs.cross_attentions, + encoder_last_hidden_state=encoder_outputs.last_hidden_state, + encoder_hidden_states=encoder_outputs.hidden_states, + encoder_attentions=encoder_outputs.attentions, + ) + + +@add_start_docstrings( + "The bare OPT Model outputting raw hidden-states without any specific head on top.", + OPT_START_DOCSTRING, +) +# Copied from transformers.models.bart.modeling_tf_bart.TFBartModel with BART->OPT,Bart->OPT +class TFOPTModel(TFOPTPretrainedModel): + + _requires_load_weight_prefix = True + + def __init__(self, config: OPTConfig, load_weight_prefix=None, *inputs, **kwargs): + super().__init__(config, *inputs, **kwargs) + + self.model = TFOPTMainLayer(config, load_weight_prefix=load_weight_prefix, name="model") + + def get_encoder(self): + return self.model.encoder + + def get_decoder(self): + return self.model.decoder + + @add_start_docstrings_to_model_forward(OPT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) + @add_code_sample_docstrings( + processor_class=_TOKENIZER_FOR_DOC, + checkpoint=_CHECKPOINT_FOR_DOC, + output_type=TFSeq2SeqModelOutput, + config_class=_CONFIG_FOR_DOC, + ) + @unpack_inputs + def call( + self, + input_ids: Optional[TFModelInputType] = None, + attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + decoder_input_ids: Optional[Union[np.ndarray, tf.Tensor]] = None, + decoder_attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + decoder_head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + cross_attn_head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + encoder_outputs: Optional[Union[Tuple, TFBaseModelOutput]] = None, + past_key_values: Optional[Tuple[Tuple[Union[np.ndarray, tf.Tensor]]]] = None, + inputs_embeds: Optional[Union[np.ndarray, tf.Tensor]] = None, + decoder_inputs_embeds: Optional[Union[np.ndarray, tf.Tensor]] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + training: Optional[bool] = False, + **kwargs + ) -> Union[TFBaseModelOutput, Tuple[tf.Tensor]]: + + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + decoder_input_ids=decoder_input_ids, + decoder_attention_mask=decoder_attention_mask, + head_mask=head_mask, + decoder_head_mask=decoder_head_mask, + cross_attn_head_mask=cross_attn_head_mask, + encoder_outputs=encoder_outputs, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + decoder_inputs_embeds=decoder_inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + training=training, + ) + + return outputs + + def serving_output(self, output): + pkv = tf.tuple(output.past_key_values)[1] if self.config.use_cache else None + dec_hs = tf.convert_to_tensor(output.decoder_hidden_states) if self.config.output_hidden_states else None + dec_attns = tf.convert_to_tensor(output.decoder_attentions) if self.config.output_attentions else None + cross_attns = tf.convert_to_tensor(output.cross_attentions) if self.config.output_attentions else None + enc_hs = tf.convert_to_tensor(output.encoder_hidden_states) if self.config.output_hidden_states else None + enc_attns = tf.convert_to_tensor(output.encoder_attentions) if self.config.output_attentions else None + + return TFSeq2SeqModelOutput( + last_hidden_state=output.last_hidden_state, + past_key_values=pkv, + decoder_hidden_states=dec_hs, + decoder_attentions=dec_attns, + cross_attentions=cross_attns, + encoder_last_hidden_state=output.encoder_last_hidden_state, + encoder_hidden_states=enc_hs, + encoder_attentions=enc_attns, + ) + + +@add_start_docstrings( + "The OPT Model with a language modeling head. Can be used for summarization.", + OPT_START_DOCSTRING, +) +# Copied from transformers.models.bart.modeling_tf_bart.TFBartForConditionalGeneration with BART->OPT,Bart->OPT +class TFOPTForConditionalGeneration(TFOPTPretrainedModel, TFCausalLanguageModelingLoss): + _keys_to_ignore_on_load_unexpected = [ + r"model.encoder.embed_tokens.weight", + r"model.decoder.embed_tokens.weight", + ] + + _requires_load_weight_prefix = True + + def __init__(self, config, load_weight_prefix=None, *inputs, **kwargs): + super().__init__(config, *inputs, **kwargs) + self.model = TFOPTMainLayer(config, load_weight_prefix=load_weight_prefix, name="model") + self.use_cache = config.use_cache + # final_bias_logits is registered as a buffer in pytorch, so not trainable for the the sake of consistency. + self.final_logits_bias = self.add_weight( + name="final_logits_bias", shape=[1, config.vocab_size], initializer="zeros", trainable=False + ) + + def get_decoder(self): + return self.model.decoder + + def get_encoder(self): + return self.model.encoder + + def get_output_embeddings(self): + return self.get_input_embeddings() + + def set_output_embeddings(self, value): + self.set_input_embeddings(value) + + def get_bias(self): + return {"final_logits_bias": self.final_logits_bias} + + def set_bias(self, value): + self.final_logits_bias = value["final_logits_bias"] + + @add_start_docstrings_to_model_forward(OPT_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=TFSeq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) + @add_end_docstrings(OPT_GENERATION_EXAMPLE) + @unpack_inputs + def call( + self, + input_ids: Optional[TFModelInputType] = None, + attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + decoder_input_ids: Optional[Union[np.ndarray, tf.Tensor]] = None, + decoder_attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + decoder_head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + cross_attn_head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, + encoder_outputs: Optional[TFBaseModelOutput] = None, + past_key_values: Optional[Tuple[Tuple[Union[np.ndarray, tf.Tensor]]]] = None, + inputs_embeds: Optional[Union[np.ndarray, tf.Tensor]] = None, + decoder_inputs_embeds: Optional[Union[np.ndarray, tf.Tensor]] = None, + use_cache: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + labels: Optional[tf.Tensor] = None, + training: Optional[bool] = False, + ) -> Union[TFSeq2SeqLMOutput, Tuple[tf.Tensor]]: + r""" + labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., + config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored + (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. + + Returns: + + """ + + if labels is not None: + labels = tf.where( + labels == self.config.pad_token_id, + tf.cast(tf.fill(shape_list(labels), -100), labels.dtype), + labels, + ) + use_cache = False + if decoder_input_ids is None: + decoder_input_ids = shift_tokens_right( + labels, self.config.pad_token_id, self.config.decoder_start_token_id + ) + + outputs = self.model( + input_ids, + attention_mask=attention_mask, + decoder_input_ids=decoder_input_ids, + encoder_outputs=encoder_outputs, + decoder_attention_mask=decoder_attention_mask, + head_mask=head_mask, + decoder_head_mask=decoder_head_mask, + cross_attn_head_mask=cross_attn_head_mask, + past_key_values=past_key_values, + inputs_embeds=inputs_embeds, + decoder_inputs_embeds=decoder_inputs_embeds, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + training=training, + ) + lm_logits = self.model.shared(outputs[0], mode="linear") + lm_logits = lm_logits + self.final_logits_bias + masked_lm_loss = None if labels is None else self.hf_compute_loss(labels, lm_logits) + + if not return_dict: + output = (lm_logits,) + outputs[1:] + return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output + return TFSeq2SeqLMOutput( + loss=masked_lm_loss, + logits=lm_logits, + past_key_values=outputs.past_key_values, # index 1 of d outputs + decoder_hidden_states=outputs.decoder_hidden_states, # index 2 of d outputs + decoder_attentions=outputs.decoder_attentions, # index 3 of d outputs + cross_attentions=outputs.cross_attentions, # index 4 of d outputs + encoder_last_hidden_state=outputs.encoder_last_hidden_state, # index 0 of encoder outputs + encoder_hidden_states=outputs.encoder_hidden_states, # 1 of e out + encoder_attentions=outputs.encoder_attentions, # 2 of e out + ) + + def serving_output(self, output): + pkv = tf.tuple(output.past_key_values)[1] if self.config.use_cache else None + dec_hs = tf.convert_to_tensor(output.decoder_hidden_states) if self.config.output_hidden_states else None + dec_attns = tf.convert_to_tensor(output.decoder_attentions) if self.config.output_attentions else None + cross_attns = tf.convert_to_tensor(output.cross_attentions) if self.config.output_attentions else None + enc_hs = tf.convert_to_tensor(output.encoder_hidden_states) if self.config.output_hidden_states else None + enc_attns = tf.convert_to_tensor(output.encoder_attentions) if self.config.output_attentions else None + + return TFSeq2SeqLMOutput( + logits=output.logits, + past_key_values=pkv, + decoder_hidden_states=dec_hs, + decoder_attentions=dec_attns, + cross_attentions=cross_attns, + encoder_last_hidden_state=output.encoder_last_hidden_state, + encoder_hidden_states=enc_hs, + encoder_attentions=enc_attns, + ) + + def prepare_inputs_for_generation( + self, + decoder_input_ids, + past=None, + attention_mask=None, + head_mask=None, + decoder_head_mask=None, + cross_attn_head_mask=None, + use_cache=None, + encoder_outputs=None, + **kwargs + ): + # cut decoder_input_ids if past is used + if past is not None: + decoder_input_ids = decoder_input_ids[:, -1:] + + return { + "input_ids": None, # encoder_outputs is defined. input_ids not needed + "encoder_outputs": encoder_outputs, + "past_key_values": past, + "decoder_input_ids": decoder_input_ids, + "attention_mask": attention_mask, + "head_mask": head_mask, + "decoder_head_mask": decoder_head_mask, + "cross_attn_head_mask": cross_attn_head_mask, + "use_cache": use_cache, # change this to avoid caching (presumably for debugging) + } + + def prepare_decoder_input_ids_from_labels(self, labels: tf.Tensor): + return shift_tokens_right(labels, self.config.pad_token_id, self.config.decoder_start_token_id) + + @staticmethod + def _reorder_cache(past, beam_idx): + reordered_past = () + for layer_past in past: + # cached cross_attention states don't have to be reordered -> they are always the same + reordered_past += ( + tuple(tf.gather(past_state, beam_idx, axis=0) for past_state in layer_past[:2]) + layer_past[2:], + ) + return reordered_past diff --git a/src/transformers/models/opt/tokenization_opt.py b/src/transformers/models/opt/tokenization_opt.py new file mode 100644 index 0000000000000..3749eef6f6351 --- /dev/null +++ b/src/transformers/models/opt/tokenization_opt.py @@ -0,0 +1,417 @@ +# coding=utf-8 +# Copyright 2020 The Facebook AI Research Team Authors and The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +from functools import lru_cache +from typing import List, Optional, Tuple + +import regex as re + +from ...tokenization_utils import AddedToken, PreTrainedTokenizer +from ...utils import logging + + +logger = logging.get_logger(__name__) + + +VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt"} + +# See all OPT models at https://huggingface.co/models?filter=opt +PRETRAINED_VOCAB_FILES_MAP = { + "vocab_file": { + "": "https://huggingface.co//resolve/main/vocab.json", + "facebook/opt-large": "https://huggingface.co/facebook/opt-large/resolve/main/vocab.json", + "facebook/opt-large-mnli": "https://huggingface.co/facebook/opt-large-mnli/resolve/main/vocab.json", + "facebook/opt-large-cnn": "https://huggingface.co/facebook/opt-large-cnn/resolve/main/vocab.json", + "facebook/opt-large-xsum": "https://huggingface.co/facebook/opt-large-xsum/resolve/main/vocab.json", + "yjernite/opt_eli5": "https://huggingface.co/yjernite/opt_eli5/resolve/main/vocab.json", + }, + "merges_file": { + "": "https://huggingface.co//resolve/main/merges.txt", + "facebook/opt-large": "https://huggingface.co/facebook/opt-large/resolve/main/merges.txt", + "facebook/opt-large-mnli": "https://huggingface.co/facebook/opt-large-mnli/resolve/main/merges.txt", + "facebook/opt-large-cnn": "https://huggingface.co/facebook/opt-large-cnn/resolve/main/merges.txt", + "facebook/opt-large-xsum": "https://huggingface.co/facebook/opt-large-xsum/resolve/main/merges.txt", + "yjernite/opt_eli5": "https://huggingface.co/yjernite/opt_eli5/resolve/main/merges.txt", + }, +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + "": 1024, + "facebook/opt-large": 1024, + "facebook/opt-large-mnli": 1024, + "facebook/opt-large-cnn": 1024, + "facebook/opt-large-xsum": 1024, + "yjernite/opt_eli5": 1024, +} + + +@lru_cache() +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control + characters the bpe code barfs on. + + The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab + if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for + decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup + tables between utf-8 bytes and unicode strings. + """ + bs = ( + list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1)) + ) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8 + n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + + +def get_pairs(word): + """ + Return set of symbol pairs in a word. + + Word is represented as tuple of symbols (symbols being variable-length strings). + """ + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + + +class OPTTokenizer(PreTrainedTokenizer): + """ + Constructs a OPT tokenizer, which is smilar to the ROBERTa tokenizer, using byte-level Byte-Pair-Encoding. + + This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will + be encoded differently whether it is at the beginning of the sentence (without space) or not: + + ``` + >>> from transformers import OPTTokenizer + >>> tokenizer = OPTTokenizer.from_pretrained("") + >>> tokenizer("Hello world")['input_ids'] + [0, 31414, 232, 2] + >>> tokenizer(" Hello world")['input_ids'] + [0, 20920, 232, 2] + ``` + + You can get around that behavior by passing `add_prefix_space=True` when instantiating this tokenizer or when you + call it on some text, but since the model was not pretrained this way, it might yield a decrease in performance. + + + + When used with `is_split_into_words=True`, this tokenizer will add a space before each word (even the first one). + + + + This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to + this superclass for more information regarding those methods. + + Args: + vocab_file (`str`): + Path to the vocabulary file. + merges_file (`str`): + Path to the merges file. + errors (`str`, *optional*, defaults to `"replace"`): + Paradigm to follow when decoding bytes to UTF-8. See + [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information. + bos_token (`str`, *optional*, defaults to `""`): + The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token. + + + + When building a sequence using special tokens, this is not the token that is used for the beginning of + sequence. The token used is the `cls_token`. + + + + eos_token (`str`, *optional*, defaults to `""`): + The end of sequence token. + + + + When building a sequence using special tokens, this is not the token that is used for the end of sequence. + The token used is the `sep_token`. + + + + sep_token (`str`, *optional*, defaults to `""`): + The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for + sequence classification or for a text and a question for question answering. It is also used as the last + token of a sequence built with special tokens. + cls_token (`str`, *optional*, defaults to `""`): + The classifier token which is used when doing sequence classification (classification of the whole sequence + instead of per-token classification). It is the first token of the sequence when built with special tokens. + unk_token (`str`, *optional*, defaults to `""`): + The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this + token instead. + pad_token (`str`, *optional*, defaults to `""`): + The token used for padding, for example when batching sequences of different lengths. + mask_token (`str`, *optional*, defaults to `""`): + The token used for masking values. This is the token used when training this model with masked language + modeling. This is the token which the model will try to predict. + add_prefix_space (`bool`, *optional*, defaults to `False`): + Whether or not to add an initial space to the input. This allows to treat the leading word just as any + other word. (OPT tokenizer detect beginning of words by the preceding space). + """ + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + model_input_names = ["input_ids", "attention_mask"] + + def __init__( + self, + vocab_file, + merges_file, + errors="replace", + bos_token="", + eos_token="", + sep_token="", + cls_token="", + unk_token="", + pad_token="", + mask_token="", + add_prefix_space=False, + **kwargs + ): + bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token + eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token + sep_token = AddedToken(sep_token, lstrip=False, rstrip=False) if isinstance(sep_token, str) else sep_token + cls_token = AddedToken(cls_token, lstrip=False, rstrip=False) if isinstance(cls_token, str) else cls_token + unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token + pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token + + # Mask token behave like a normal word, i.e. include the space before it + mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token + + super().__init__( + errors=errors, + bos_token=bos_token, + eos_token=eos_token, + unk_token=unk_token, + sep_token=sep_token, + cls_token=cls_token, + pad_token=pad_token, + mask_token=mask_token, + add_prefix_space=add_prefix_space, + **kwargs, + ) + + with open(vocab_file, encoding="utf-8") as vocab_handle: + self.encoder = json.load(vocab_handle) + self.decoder = {v: k for k, v in self.encoder.items()} + self.errors = errors # how to handle errors in decoding + self.byte_encoder = bytes_to_unicode() + self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} + with open(merges_file, encoding="utf-8") as merges_handle: + bpe_merges = merges_handle.read().split("\n")[1:-1] + bpe_merges = [tuple(merge.split()) for merge in bpe_merges] + self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges)))) + self.cache = {} + self.add_prefix_space = add_prefix_space + + # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions + self.pat = re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""") + + @property + def vocab_size(self): + return len(self.encoder) + + def get_vocab(self): + return dict(self.encoder, **self.added_tokens_encoder) + + def bpe(self, token): + if token in self.cache: + return self.cache[token] + word = tuple(token) + pairs = get_pairs(word) + + if not pairs: + return token + + while True: + bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf"))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + while i < len(word): + try: + j = word.index(first, i) + except ValueError: + new_word.extend(word[i:]) + break + else: + new_word.extend(word[i:j]) + i = j + + if word[i] == first and i < len(word) - 1 and word[i + 1] == second: + new_word.append(first + second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + else: + pairs = get_pairs(word) + word = " ".join(word) + self.cache[token] = word + return word + + def _tokenize(self, text): + """Tokenize a string.""" + bpe_tokens = [] + for token in re.findall(self.pat, text): + token = "".join( + self.byte_encoder[b] for b in token.encode("utf-8") + ) # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case) + bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(" ")) + return bpe_tokens + + def _convert_token_to_id(self, token): + """Converts a token (str) in an id using the vocab.""" + return self.encoder.get(token, self.encoder.get(self.unk_token)) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (str) using the vocab.""" + return self.decoder.get(index) + + def convert_tokens_to_string(self, tokens): + """Converts a sequence of tokens (string) in a single string.""" + text = "".join(tokens) + text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors=self.errors) + return text + + def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]: + if not os.path.isdir(save_directory): + logger.error(f"Vocabulary path ({save_directory}) should be a directory") + return + vocab_file = os.path.join( + save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] + ) + merge_file = os.path.join( + save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"] + ) + + with open(vocab_file, "w", encoding="utf-8") as f: + f.write(json.dumps(self.encoder, ensure_ascii=False)) + + index = 0 + with open(merge_file, "w", encoding="utf-8") as writer: + writer.write("#version: 0.2\n") + for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]): + if index != token_index: + logger.warning( + f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive." + " Please check that the tokenizer is not corrupted!" + ) + index = token_index + writer.write(" ".join(bpe_tokens) + "\n") + index += 1 + + return vocab_file, merge_file + + def build_inputs_with_special_tokens( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and + adding special tokens. A OPT sequence has the following format: + + - single sequence: ` X ` + - pair of sequences: ` A B ` + + Args: + token_ids_0 (`List[int]`): + List of IDs to which the special tokens will be added. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens. + """ + if token_ids_1 is None: + return [self.cls_token_id] + token_ids_0 + [self.sep_token_id] + cls = [self.cls_token_id] + sep = [self.sep_token_id] + return cls + token_ids_0 + sep + sep + token_ids_1 + sep + + def get_special_tokens_mask( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False + ) -> List[int]: + """ + Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer `prepare_for_model` method. + + Args: + token_ids_0 (`List[int]`): + List of IDs. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + already_has_special_tokens (`bool`, *optional*, defaults to `False`): + Whether or not the token list is already formatted with special tokens for the model. + + Returns: + `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. + """ + if already_has_special_tokens: + return super().get_special_tokens_mask( + token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True + ) + + if token_ids_1 is None: + return [1] + ([0] * len(token_ids_0)) + [1] + return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1] + + def create_token_type_ids_from_sequences( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """ + Create a mask from the two sequences passed to be used in a sequence-pair classification task. OPT does not + make use of token type ids, therefore a list of zeros is returned. + + Args: + token_ids_0 (`List[int]`): + List of IDs. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of zeros. + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + + if token_ids_1 is None: + return len(cls + token_ids_0 + sep) * [0] + return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0] + + def prepare_for_tokenization(self, text, is_split_into_words=False, **kwargs): + add_prefix_space = kwargs.pop("add_prefix_space", self.add_prefix_space) + if (is_split_into_words or add_prefix_space) and (len(text) > 0 and not text[0].isspace()): + text = " " + text + return (text, kwargs) diff --git a/src/transformers/models/opt/tokenization_opt_fast.py b/src/transformers/models/opt/tokenization_opt_fast.py new file mode 100644 index 0000000000000..429baa552cc29 --- /dev/null +++ b/src/transformers/models/opt/tokenization_opt_fast.py @@ -0,0 +1,303 @@ +# coding=utf-8 +# Copyright 2020 The Facebook AI Research Team Authors and The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from typing import List, Optional, Tuple + +from tokenizers import pre_tokenizers, processors + +from ...tokenization_utils_base import AddedToken, BatchEncoding +from ...tokenization_utils_fast import PreTrainedTokenizerFast +from ...utils import logging +from .tokenization_opt import OPTTokenizer + + +logger = logging.get_logger(__name__) + + +VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"} + +# See all OPT models at https://huggingface.co/models?filter=opt +PRETRAINED_VOCAB_FILES_MAP = { + "vocab_file": { + "": "https://huggingface.co//resolve/main/vocab.json", + "facebook/opt-large": "https://huggingface.co/facebook/opt-large/resolve/main/vocab.json", + "facebook/opt-large-mnli": "https://huggingface.co/facebook/opt-large-mnli/resolve/main/vocab.json", + "facebook/opt-large-cnn": "https://huggingface.co/facebook/opt-large-cnn/resolve/main/vocab.json", + "facebook/opt-large-xsum": "https://huggingface.co/facebook/opt-large-xsum/resolve/main/vocab.json", + "yjernite/opt_eli5": "https://huggingface.co/yjernite/opt_eli5/resolve/main/vocab.json", + }, + "merges_file": { + "": "https://huggingface.co//resolve/main/merges.txt", + "facebook/opt-large": "https://huggingface.co/facebook/opt-large/resolve/main/merges.txt", + "facebook/opt-large-mnli": "https://huggingface.co/facebook/opt-large-mnli/resolve/main/merges.txt", + "facebook/opt-large-cnn": "https://huggingface.co/facebook/opt-large-cnn/resolve/main/merges.txt", + "facebook/opt-large-xsum": "https://huggingface.co/facebook/opt-large-xsum/resolve/main/merges.txt", + "yjernite/opt_eli5": "https://huggingface.co/yjernite/opt_eli5/resolve/main/merges.txt", + }, + "tokenizer_file": { + "": "https://huggingface.co//resolve/main/tokenizer.json", + "facebook/opt-large": "https://huggingface.co/facebook/opt-large/resolve/main/tokenizer.json", + "facebook/opt-large-mnli": "https://huggingface.co/facebook/opt-large-mnli/resolve/main/tokenizer.json", + "facebook/opt-large-cnn": "https://huggingface.co/facebook/opt-large-cnn/resolve/main/tokenizer.json", + "facebook/opt-large-xsum": "https://huggingface.co/facebook/opt-large-xsum/resolve/main/tokenizer.json", + "yjernite/opt_eli5": "https://huggingface.co/yjernite/opt_eli5/resolve/main/tokenizer.json", + }, +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + "": 1024, + "facebook/opt-large": 1024, + "facebook/opt-large-mnli": 1024, + "facebook/opt-large-cnn": 1024, + "facebook/opt-large-xsum": 1024, + "yjernite/opt_eli5": 1024, +} + + +class OPTTokenizerFast(PreTrainedTokenizerFast): + r""" + Construct a "fast" OPT tokenizer (backed by HuggingFace's *tokenizers* library), derived from the GPT-2 tokenizer, + using byte-level Byte-Pair-Encoding. + + This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will + be encoded differently whether it is at the beginning of the sentence (without space) or not: + + ``` + >>> from transformers import OPTTokenizerFast + >>> tokenizer = OPTTokenizerFast.from_pretrained("") + >>> tokenizer("Hello world")['input_ids'] + [0, 31414, 232, 2] + >>> tokenizer(" Hello world")['input_ids'] + [0, 20920, 232, 2] + ``` + + You can get around that behavior by passing `add_prefix_space=True` when instantiating this tokenizer or when you + call it on some text, but since the model was not pretrained this way, it might yield a decrease in performance. + + + + When used with `is_split_into_words=True`, this tokenizer needs to be instantiated with `add_prefix_space=True`. + + + + This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should + refer to this superclass for more information regarding those methods. + + Args: + vocab_file (`str`): + Path to the vocabulary file. + merges_file (`str`): + Path to the merges file. + errors (`str`, *optional*, defaults to `"replace"`): + Paradigm to follow when decoding bytes to UTF-8. See + [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information. + bos_token (`str`, *optional*, defaults to `""`): + The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token. + + + + When building a sequence using special tokens, this is not the token that is used for the beginning of + sequence. The token used is the `cls_token`. + + + + eos_token (`str`, *optional*, defaults to `""`): + The end of sequence token. + + + + When building a sequence using special tokens, this is not the token that is used for the end of sequence. + The token used is the `sep_token`. + + + + sep_token (`str`, *optional*, defaults to `""`): + The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for + sequence classification or for a text and a question for question answering. It is also used as the last + token of a sequence built with special tokens. + cls_token (`str`, *optional*, defaults to `""`): + The classifier token which is used when doing sequence classification (classification of the whole sequence + instead of per-token classification). It is the first token of the sequence when built with special tokens. + unk_token (`str`, *optional*, defaults to `""`): + The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this + token instead. + pad_token (`str`, *optional*, defaults to `""`): + The token used for padding, for example when batching sequences of different lengths. + mask_token (`str`, *optional*, defaults to `""`): + The token used for masking values. This is the token used when training this model with masked language + modeling. This is the token which the model will try to predict. + add_prefix_space (`bool`, *optional*, defaults to `False`): + Whether or not to add an initial space to the input. This allows to treat the leading word just as any + other word. (OPT tokenizer detect beginning of words by the preceding space). + trim_offsets (`bool`, *optional*, defaults to `True`): + Whether the post processing step should trim offsets to avoid including whitespaces. + """ + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + model_input_names = ["input_ids", "attention_mask"] + slow_tokenizer_class = OPTTokenizer + + def __init__( + self, + vocab_file=None, + merges_file=None, + tokenizer_file=None, + errors="replace", + bos_token="", + eos_token="", + sep_token="", + cls_token="", + unk_token="", + pad_token="", + mask_token="", + add_prefix_space=False, + trim_offsets=True, + **kwargs + ): + super().__init__( + vocab_file, + merges_file, + tokenizer_file=tokenizer_file, + errors=errors, + bos_token=bos_token, + eos_token=eos_token, + sep_token=sep_token, + cls_token=cls_token, + unk_token=unk_token, + pad_token=pad_token, + mask_token=mask_token, + add_prefix_space=add_prefix_space, + trim_offsets=trim_offsets, + **kwargs, + ) + + pre_tok_state = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()) + if pre_tok_state.get("add_prefix_space", add_prefix_space) != add_prefix_space: + pre_tok_class = getattr(pre_tokenizers, pre_tok_state.pop("type")) + pre_tok_state["add_prefix_space"] = add_prefix_space + self.backend_tokenizer.pre_tokenizer = pre_tok_class(**pre_tok_state) + + self.add_prefix_space = add_prefix_space + + # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__` + tokenizer_component = "post_processor" + tokenizer_component_instance = getattr(self.backend_tokenizer, tokenizer_component, None) + if tokenizer_component_instance: + state = json.loads(tokenizer_component_instance.__getstate__()) + + # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class` + if "sep" in state: + state["sep"] = tuple(state["sep"]) + if "cls" in state: + state["cls"] = tuple(state["cls"]) + + changes_to_apply = False + + if state.get("add_prefix_space", add_prefix_space) != add_prefix_space: + state["add_prefix_space"] = add_prefix_space + changes_to_apply = True + + if state.get("trim_offsets", trim_offsets) != trim_offsets: + state["trim_offsets"] = trim_offsets + changes_to_apply = True + + if changes_to_apply: + component_class = getattr(processors, state.pop("type")) + new_value = component_class(**state) + setattr(self.backend_tokenizer, tokenizer_component, new_value) + + @property + def mask_token(self) -> str: + """ + `str`: Mask token, to use when training a model with masked-language modeling. Log an error if used while not + having been set. + + OPT tokenizer has a special mask token to be usable in the fill-mask pipeline. The mask token will greedily + comprise the space before the **. + """ + if self._mask_token is None and self.verbose: + logger.error("Using mask_token, but it is not set yet.") + return None + return str(self._mask_token) + + @mask_token.setter + def mask_token(self, value): + """ + Overriding the default behavior of the mask token to have it eat the space before it. + + This is needed to preserve backward compatibility with all the previously used models based on OPT. + """ + # Mask token behave like a normal word, i.e. include the space before it + # So we set lstrip to True + value = AddedToken(value, lstrip=True, rstrip=False) if isinstance(value, str) else value + self._mask_token = value + + def _batch_encode_plus(self, *args, **kwargs) -> BatchEncoding: + is_split_into_words = kwargs.get("is_split_into_words", False) + + if is_split_into_words and not self.add_prefix_space: + raise ValueError( + f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True " + "to use it with pretokenized inputs." + ) + + return super()._batch_encode_plus(*args, **kwargs) + + def _encode_plus(self, *args, **kwargs) -> BatchEncoding: + is_split_into_words = kwargs.get("is_split_into_words", False) + + if is_split_into_words and not self.add_prefix_space: + raise ValueError( + f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True " + "to use it with pretokenized inputs." + ) + + return super()._encode_plus(*args, **kwargs) + + def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]: + files = self._tokenizer.model.save(save_directory, name=filename_prefix) + return tuple(files) + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + output = [self.bos_token_id] + token_ids_0 + [self.eos_token_id] + if token_ids_1 is None: + return output + + return output + [self.eos_token_id] + token_ids_1 + [self.eos_token_id] + + def create_token_type_ids_from_sequences( + self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None + ) -> List[int]: + """ + Create a mask from the two sequences passed to be used in a sequence-pair classification task. OPT does not + make use of token type ids, therefore a list of zeros is returned. + + Args: + token_ids_0 (`List[int]`): + List of IDs. + token_ids_1 (`List[int]`, *optional*): + Optional second list of IDs for sequence pairs. + + Returns: + `List[int]`: List of zeros. + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + + if token_ids_1 is None: + return len(cls + token_ids_0 + sep) * [0] + return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0] diff --git a/tests/models/opt/__init__.py b/tests/models/opt/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/models/opt/test_embeddings_opt.py b/tests/models/opt/test_embeddings_opt.py new file mode 100644 index 0000000000000..fc179f559060a --- /dev/null +++ b/tests/models/opt/test_embeddings_opt.py @@ -0,0 +1,66 @@ +# coding=utf-8 +# Copyright 2021, The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Testing suite for the PyTorch OPT model. """ + +import unittest + + +from transformers import OPTConfig, is_torch_available, BartTokenizerFast + + + +if is_torch_available(): + import torch + + from transformers import ( + AutoModelForSequenceClassification, + OPTForCausalLM, + OPTForConditionalGeneration, + OPTForQuestionAnswering, + OPTForSequenceClassification, + OPTModel, + OPTTokenizer, + pipeline, + ) + +class OPTEmbeddingsTest(unittest.TestCase): + def setUp(self): + super().setUp() + self.path_model = "/home/younes/Desktop/Work/data/opt-350-m/" + self.path_logits_meta = "/home/younes/Desktop/Work/metaseq-conversion/logits_metaseq.p" + + def test_load_model(self): + try: + _ = OPTForCausalLM.from_pretrained(self.path_model) + except BaseException: + self.fail("Failed loading model") + + def test_logits(self): + model = OPTForCausalLM.from_pretrained(self.path_model) + model = model.eval() + tokenizer = BartTokenizerFast.from_pretrained("facebook/bart-large") + prompts = [ + "Today is a beautiful day and I want to", + "In the city of", + "Paris is the capital of France and", + "Computers and mobile phones have taken", + ] + input_ids = tokenizer(prompts, return_tensors="pt", padding=True, truncation=True).input_ids + logits = model(input_ids)[0] + logits_meta = torch.load(self.path_logits_meta) + assert torch.allclose(logits, logits_meta.permute(1, 0, 2)) + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/models/opt/test_modeling_flax_opt.py b/tests/models/opt/test_modeling_flax_opt.py new file mode 100644 index 0000000000000..10e8fdf6c57f3 --- /dev/null +++ b/tests/models/opt/test_modeling_flax_opt.py @@ -0,0 +1,567 @@ +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +import numpy as np +import timeout_decorator # noqa + +from transformers import OPTConfig, OPTTokenizer, is_flax_available +from transformers.testing_utils import require_flax, slow + +from ...generation.test_generation_flax_utils import FlaxGenerationTesterMixin +from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask + + +if is_flax_available(): + import os + + # The slow tests are often failing with OOM error on GPU + # This makes JAX allocate exactly what is needed on demand, and deallocate memory that is no longer needed + # but will be slower as stated here https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html + os.environ["XLA_PYTHON_CLIENT_ALLOCATOR"] = "platform" + + import jax + import jax.numpy as jnp + from transformers.models.opt.modeling_flax_opt import ( + FlaxOPTForConditionalGeneration, + FlaxOPTForQuestionAnswering, + FlaxOPTForSequenceClassification, + FlaxOPTModel, + shift_tokens_right, + ) + + +def prepare_opt_inputs_dict( + config, + input_ids, + decoder_input_ids=None, + attention_mask=None, + decoder_attention_mask=None, + head_mask=None, + decoder_head_mask=None, + cross_attn_head_mask=None, +): + if attention_mask is None: + attention_mask = np.where(input_ids != config.pad_token_id, 1, 0) + if decoder_attention_mask is None: + decoder_attention_mask = np.where(decoder_input_ids != config.pad_token_id, 1, 0) + if head_mask is None: + head_mask = np.ones((config.encoder_layers, config.encoder_attention_heads)) + if decoder_head_mask is None: + decoder_head_mask = np.ones((config.decoder_layers, config.decoder_attention_heads)) + if cross_attn_head_mask is None: + cross_attn_head_mask = np.ones((config.decoder_layers, config.decoder_attention_heads)) + return { + "input_ids": input_ids, + "decoder_input_ids": decoder_input_ids, + "attention_mask": attention_mask, + "decoder_attention_mask": attention_mask, + } + + +class FlaxOPTModelTester: + def __init__( + self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_labels=False, + vocab_size=99, + hidden_size=16, + num_hidden_layers=2, + num_attention_heads=4, + intermediate_size=4, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=32, + eos_token_id=2, + pad_token_id=1, + bos_token_id=0, + initializer_range=0.02, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_labels = use_labels + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.eos_token_id = eos_token_id + self.pad_token_id = pad_token_id + self.bos_token_id = bos_token_id + self.initializer_range = initializer_range + + def prepare_config_and_inputs(self): + input_ids = np.clip(ids_tensor([self.batch_size, self.seq_length - 1], self.vocab_size), 3, self.vocab_size) + input_ids = np.concatenate((input_ids, 2 * np.ones((self.batch_size, 1), dtype=np.int64)), -1) + + decoder_input_ids = shift_tokens_right(input_ids, 1, 2) + + config = OPTConfig( + vocab_size=self.vocab_size, + d_model=self.hidden_size, + encoder_layers=self.num_hidden_layers, + decoder_layers=self.num_hidden_layers, + encoder_attention_heads=self.num_attention_heads, + decoder_attention_heads=self.num_attention_heads, + encoder_ffn_dim=self.intermediate_size, + decoder_ffn_dim=self.intermediate_size, + dropout=self.hidden_dropout_prob, + attention_dropout=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + eos_token_id=self.eos_token_id, + bos_token_id=self.bos_token_id, + pad_token_id=self.pad_token_id, + initializer_range=self.initializer_range, + use_cache=False, + ) + inputs_dict = prepare_opt_inputs_dict(config, input_ids, decoder_input_ids) + return config, inputs_dict + + def prepare_config_and_inputs_for_common(self): + config, inputs_dict = self.prepare_config_and_inputs() + return config, inputs_dict + + def check_use_cache_forward(self, model_class_name, config, inputs_dict): + max_decoder_length = 20 + model = model_class_name(config) + + encoder_outputs = model.encode(inputs_dict["input_ids"]) + + decoder_input_ids, decoder_attention_mask = ( + inputs_dict["decoder_input_ids"], + inputs_dict["decoder_attention_mask"], + ) + + past_key_values = model.init_cache(decoder_input_ids.shape[0], max_decoder_length, encoder_outputs) + decoder_attention_mask = jnp.ones((decoder_input_ids.shape[0], max_decoder_length), dtype="i4") + + decoder_position_ids = jnp.broadcast_to( + jnp.arange(decoder_input_ids.shape[-1] - 1)[None, :], + (decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1), + ) + outputs_cache = model.decode( + decoder_input_ids[:, :-1], + encoder_outputs, + decoder_attention_mask=decoder_attention_mask, + past_key_values=past_key_values, + decoder_position_ids=decoder_position_ids, + ) + + decoder_position_ids = jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]], dtype="i4") + outputs_cache_next = model.decode( + decoder_input_ids[:, -1:], + encoder_outputs, + decoder_attention_mask=decoder_attention_mask, + past_key_values=outputs_cache.past_key_values, + decoder_position_ids=decoder_position_ids, + ) + + outputs = model.decode(decoder_input_ids, encoder_outputs) + + diff = np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5]))) + self.parent.assertTrue(diff < 1e-3, msg=f"Max diff is {diff}") + + def check_use_cache_forward_with_attn_mask(self, model_class_name, config, inputs_dict): + max_decoder_length = 20 + model = model_class_name(config) + + encoder_outputs = model.encode(inputs_dict["input_ids"]) + + decoder_input_ids, decoder_attention_mask = ( + inputs_dict["decoder_input_ids"], + inputs_dict["decoder_attention_mask"], + ) + + decoder_attention_mask_cache = jnp.concatenate( + [ + decoder_attention_mask, + jnp.zeros((decoder_attention_mask.shape[0], max_decoder_length - decoder_attention_mask.shape[1])), + ], + axis=-1, + ) + + past_key_values = model.init_cache(decoder_input_ids.shape[0], max_decoder_length, encoder_outputs) + decoder_position_ids = jnp.broadcast_to( + jnp.arange(decoder_input_ids.shape[-1] - 1)[None, :], + (decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1), + ) + + outputs_cache = model.decode( + decoder_input_ids[:, :-1], + encoder_outputs, + decoder_attention_mask=decoder_attention_mask_cache, + past_key_values=past_key_values, + decoder_position_ids=decoder_position_ids, + ) + decoder_position_ids = jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]], dtype="i4") + outputs_cache_next = model.decode( + decoder_input_ids[:, -1:], + encoder_outputs, + past_key_values=outputs_cache.past_key_values, + decoder_attention_mask=decoder_attention_mask_cache, + decoder_position_ids=decoder_position_ids, + ) + + outputs = model.decode(decoder_input_ids, encoder_outputs, decoder_attention_mask=decoder_attention_mask) + + diff = np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5]))) + self.parent.assertTrue(diff < 1e-3, msg=f"Max diff is {diff}") + + +@require_flax +class OPTHeadTests(unittest.TestCase): + vocab_size = 99 + + def _get_config_and_data(self): + input_ids = np.array( + [ + [71, 82, 18, 33, 46, 91, 2], + [68, 34, 26, 58, 30, 82, 2], + [5, 97, 17, 39, 94, 40, 2], + [76, 83, 94, 25, 70, 78, 2], + [87, 59, 41, 35, 48, 66, 2], + [55, 13, 16, 58, 5, 2, 1], # note padding + [64, 27, 31, 51, 12, 75, 2], + [52, 64, 86, 17, 83, 39, 2], + [48, 61, 9, 24, 71, 82, 2], + [26, 1, 60, 48, 22, 13, 2], + [21, 5, 62, 28, 14, 76, 2], + [45, 98, 37, 86, 59, 48, 2], + [70, 70, 50, 9, 28, 0, 2], + ], + dtype=np.int64, + ) + + batch_size = input_ids.shape[0] + config = OPTConfig( + vocab_size=self.vocab_size, + d_model=24, + encoder_layers=2, + decoder_layers=2, + encoder_attention_heads=2, + decoder_attention_heads=2, + encoder_ffn_dim=32, + decoder_ffn_dim=32, + max_position_embeddings=48, + eos_token_id=2, + pad_token_id=1, + bos_token_id=0, + ) + return config, input_ids, batch_size + + def test_sequence_classification_forward(self): + config, input_ids, batch_size = self._get_config_and_data() + model = FlaxOPTForSequenceClassification(config) + outputs = model(input_ids=input_ids, decoder_input_ids=input_ids) + expected_shape = (batch_size, config.num_labels) + self.assertEqual(outputs["logits"].shape, expected_shape) + + def test_question_answering_forward(self): + config, input_ids, batch_size = self._get_config_and_data() + model = FlaxOPTForQuestionAnswering(config) + outputs = model(input_ids=input_ids) + + self.assertEqual(outputs["start_logits"].shape, input_ids.shape) + self.assertEqual(outputs["end_logits"].shape, input_ids.shape) + + # @timeout_decorator.timeout(1) # not working with the decorator so far + def test_lm_forward(self): + config, input_ids, batch_size = self._get_config_and_data() + lm_model = FlaxOPTForConditionalGeneration(config) + outputs = lm_model(input_ids=input_ids) + expected_shape = (batch_size, input_ids.shape[1], config.vocab_size) + self.assertEqual(outputs["logits"].shape, expected_shape) + + def test_lm_uneven_forward(self): + config = OPTConfig( + vocab_size=self.vocab_size, + d_model=14, + encoder_layers=2, + decoder_layers=2, + encoder_attention_heads=2, + decoder_attention_heads=2, + encoder_ffn_dim=8, + decoder_ffn_dim=8, + max_position_embeddings=48, + ) + lm_model = FlaxOPTForConditionalGeneration(config) + context = np.array([[71, 82, 18, 33, 46, 91, 2], [68, 34, 26, 58, 30, 2, 1]], dtype=np.int64) + summary = np.array([[82, 71, 82, 18, 2], [58, 68, 2, 1, 1]], dtype=np.int64) + outputs = lm_model(input_ids=context, decoder_input_ids=summary) + expected_shape = (*summary.shape, config.vocab_size) + self.assertEqual(outputs["logits"].shape, expected_shape) + + def test_shift_tokens_right(self): + input_ids = np.array([[71, 82, 18, 33, 2, 1, 1], [68, 34, 26, 58, 30, 82, 2]], dtype=np.int64) + shifted = shift_tokens_right(input_ids, 1, 2) + n_pad_before = np.equal(input_ids, 1).astype(np.float32).sum() + n_pad_after = np.equal(shifted, 1).astype(np.float32).sum() + self.assertEqual(shifted.shape, input_ids.shape) + self.assertEqual(n_pad_after, n_pad_before - 1) + self.assertTrue(np.equal(shifted[:, 0], 2).all()) + + +@require_flax +class FlaxOPTModelTest(FlaxModelTesterMixin, unittest.TestCase, FlaxGenerationTesterMixin): + is_encoder_decoder = True + all_model_classes = ( + ( + FlaxOPTModel, + FlaxOPTForConditionalGeneration, + FlaxOPTForSequenceClassification, + FlaxOPTForQuestionAnswering, + ) + if is_flax_available() + else () + ) + all_generative_model_classes = (FlaxOPTForConditionalGeneration,) if is_flax_available() else () + + def setUp(self): + self.model_tester = FlaxOPTModelTester(self) + + def test_use_cache_forward(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs() + for model_class in self.all_model_classes: + self.model_tester.check_use_cache_forward(model_class, config, inputs_dict) + + def test_use_cache_forward_with_attn_mask(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs() + for model_class in self.all_model_classes: + self.model_tester.check_use_cache_forward_with_attn_mask(model_class, config, inputs_dict) + + def test_encode(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + with self.subTest(model_class.__name__): + prepared_inputs_dict = self._prepare_for_class(inputs_dict, model_class) + model = model_class(config) + + @jax.jit + def encode_jitted(input_ids, attention_mask=None, **kwargs): + return model.encode(input_ids=input_ids, attention_mask=attention_mask) + + with self.subTest("JIT Enabled"): + jitted_outputs = encode_jitted(**prepared_inputs_dict).to_tuple() + + with self.subTest("JIT Disabled"): + with jax.disable_jit(): + outputs = encode_jitted(**prepared_inputs_dict).to_tuple() + + self.assertEqual(len(outputs), len(jitted_outputs)) + for jitted_output, output in zip(jitted_outputs, outputs): + self.assertEqual(jitted_output.shape, output.shape) + + def test_decode(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + with self.subTest(model_class.__name__): + model = model_class(config) + encoder_outputs = model.encode(inputs_dict["input_ids"], inputs_dict["attention_mask"]) + + prepared_inputs_dict = { + "decoder_input_ids": inputs_dict["decoder_input_ids"], + "decoder_attention_mask": inputs_dict["decoder_attention_mask"], + "encoder_outputs": encoder_outputs, + } + + @jax.jit + def decode_jitted(decoder_input_ids, decoder_attention_mask, encoder_outputs): + return model.decode( + decoder_input_ids=decoder_input_ids, + decoder_attention_mask=decoder_attention_mask, + encoder_outputs=encoder_outputs, + ) + + with self.subTest("JIT Enabled"): + jitted_outputs = decode_jitted(**prepared_inputs_dict).to_tuple() + + with self.subTest("JIT Disabled"): + with jax.disable_jit(): + outputs = decode_jitted(**prepared_inputs_dict).to_tuple() + + self.assertEqual(len(outputs), len(jitted_outputs)) + for jitted_output, output in zip(jitted_outputs, outputs): + self.assertEqual(jitted_output.shape, output.shape) + + @slow + def test_model_from_pretrained(self): + for model_class_name in self.all_model_classes: + model = model_class_name.from_pretrained("", from_pt=True) + # FlaxOPTForSequenceClassification expects eos token in input_ids + input_ids = np.ones((1, 1)) * model.config.eos_token_id + outputs = model(input_ids) + self.assertIsNotNone(outputs) + + @slow + def test_summarization_fast(self): + model = FlaxOPTForConditionalGeneration.from_pretrained("sshleifer/distilopt-cnn-6-6") + tokenizer = OPTTokenizer.from_pretrained("sshleifer/distilopt-cnn-6-6") + + input_str = "This sentence is made of three parts. Each part is important on its own. One part is about animals, the other part about planes, and the last part about housing." + + input_ids = tokenizer(input_str, return_tensors="np").input_ids + sequences = model.generate(input_ids, num_beams=2, max_length=20).sequences + + output_str = tokenizer.batch_decode(sequences)[0] + + assert ( + output_str == "This sentence is made of three parts. One part is about animals, the other part" + ) + + @slow + def test_cnn_summarization_same_as_fairseq(self): + model = FlaxOPTForConditionalGeneration.from_pretrained("facebook/opt-large-cnn") + tokenizer = OPTTokenizer.from_pretrained("facebook/opt-large-cnn") + + FRANCE_ARTICLE = ' Marseille, France (CNN)The French prosecutor leading an investigation into the crash of Germanwings Flight 9525 insisted Wednesday that he was not aware of any video footage from on board the plane. Marseille prosecutor Brice Robin told CNN that "so far no videos were used in the crash investigation." He added, "A person who has such a video needs to immediately give it to the investigators." Robin\'s comments follow claims by two magazines, German daily Bild and French Paris Match, of a cell phone video showing the harrowing final seconds from on board Germanwings Flight 9525 as it crashed into the French Alps. All 150 on board were killed. Paris Match and Bild reported that the video was recovered from a phone at the wreckage site. The two publications described the supposed video, but did not post it on their websites. The publications said that they watched the video, which was found by a source close to the investigation. "One can hear cries of \'My God\' in several languages," Paris Match reported. "Metallic banging can also be heard more than three times, perhaps of the pilot trying to open the cockpit door with a heavy object. Towards the end, after a heavy shake, stronger than the others, the screaming intensifies. Then nothing." "It is a very disturbing scene," said Julian Reichelt, editor-in-chief of Bild online. An official with France\'s accident investigation agency, the BEA, said the agency is not aware of any such video. Lt. Col. Jean-Marc Menichini, a French Gendarmerie spokesman in charge of communications on rescue efforts around the Germanwings crash site, told CNN that the reports were "completely wrong" and "unwarranted." Cell phones have been collected at the site, he said, but that they "hadn\'t been exploited yet." Menichini said he believed the cell phones would need to be sent to the Criminal Research Institute in Rosny sous-Bois, near Paris, in order to be analyzed by specialized technicians working hand-in-hand with investigators. But none of the cell phones found so far have been sent to the institute, Menichini said. Asked whether staff involved in the search could have leaked a memory card to the media, Menichini answered with a categorical "no." Reichelt told "Erin Burnett: Outfront" that he had watched the video and stood by the report, saying Bild and Paris Match are "very confident" that the clip is real. He noted that investigators only revealed they\'d recovered cell phones from the crash site after Bild and Paris Match published their reports. "That is something we did not know before. ... Overall we can say many things of the investigation weren\'t revealed by the investigation at the beginning," he said. What was mental state of Germanwings co-pilot? German airline Lufthansa confirmed Tuesday that co-pilot Andreas Lubitz had battled depression years before he took the controls of Germanwings Flight 9525, which he\'s accused of deliberately crashing last week in the French Alps. Lubitz told his Lufthansa flight training school in 2009 that he had a "previous episode of severe depression," the airline said Tuesday. Email correspondence between Lubitz and the school discovered in an internal investigation, Lufthansa said, included medical documents he submitted in connection with resuming his flight training. The announcement indicates that Lufthansa, the parent company of Germanwings, knew of Lubitz\'s battle with depression, allowed him to continue training and ultimately put him in the cockpit. Lufthansa, whose CEO Carsten Spohr previously said Lubitz was 100% fit to fly, described its statement Tuesday as a "swift and seamless clarification" and said it was sharing the information and documents -- including training and medical records -- with public prosecutors. Spohr traveled to the crash site Wednesday, where recovery teams have been working for the past week to recover human remains and plane debris scattered across a steep mountainside. He saw the crisis center set up in Seyne-les-Alpes, laid a wreath in the village of Le Vernet, closer to the crash site, where grieving families have left flowers at a simple stone memorial. Menichini told CNN late Tuesday that no visible human remains were left at the site but recovery teams would keep searching. French President Francois Hollande, speaking Tuesday, said that it should be possible to identify all the victims using DNA analysis by the end of the week, sooner than authorities had previously suggested. In the meantime, the recovery of the victims\' personal belongings will start Wednesday, Menichini said. Among those personal belongings could be more cell phones belonging to the 144 passengers and six crew on board. Check out the latest from our correspondents . The details about Lubitz\'s correspondence with the flight school during his training were among several developments as investigators continued to delve into what caused the crash and Lubitz\'s possible motive for downing the jet. A Lufthansa spokesperson told CNN on Tuesday that Lubitz had a valid medical certificate, had passed all his examinations and "held all the licenses required." Earlier, a spokesman for the prosecutor\'s office in Dusseldorf, Christoph Kumpa, said medical records reveal Lubitz suffered from suicidal tendencies at some point before his aviation career and underwent psychotherapy before he got his pilot\'s license. Kumpa emphasized there\'s no evidence suggesting Lubitz was suicidal or acting aggressively before the crash. Investigators are looking into whether Lubitz feared his medical condition would cause him to lose his pilot\'s license, a European government official briefed on the investigation told CNN on Tuesday. While flying was "a big part of his life," the source said, it\'s only one theory being considered. Another source, a law enforcement official briefed on the investigation, also told CNN that authorities believe the primary motive for Lubitz to bring down the plane was that he feared he would not be allowed to fly because of his medical problems. Lubitz\'s girlfriend told investigators he had seen an eye doctor and a neuropsychologist, both of whom deemed him unfit to work recently and concluded he had psychological issues, the European government official said. But no matter what details emerge about his previous mental health struggles, there\'s more to the story, said Brian Russell, a forensic psychologist. "Psychology can explain why somebody would turn rage inward on themselves about the fact that maybe they weren\'t going to keep doing their job and they\'re upset about that and so they\'re suicidal," he said. "But there is no mental illness that explains why somebody then feels entitled to also take that rage and turn it outward on 149 other people who had nothing to do with the person\'s problems." Germanwings crash compensation: What we know . Who was the captain of Germanwings Flight 9525? CNN\'s Margot Haddad reported from Marseille and Pamela Brown from Dusseldorf, while Laura Smith-Spark wrote from London. CNN\'s Frederik Pleitgen, Pamela Boykoff, Antonia Mortensen, Sandrine Amiel and Anna-Maja Rappard contributed to this report.' # @noq + + SHORTER_ARTICLE = ' (CNN)The Palestinian Authority officially became the 123rd member of the International Criminal Court on Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian territories. The formal accession was marked with a ceremony at The Hague, in the Netherlands, where the court is based. The Palestinians signed the ICC\'s founding Rome Statute in January, when they also accepted its jurisdiction over alleged crimes committed "in the occupied Palestinian territory, including East Jerusalem, since June 13, 2014." Later that month, the ICC opened a preliminary examination into the situation in Palestinian territories, paving the way for possible war crimes investigations against Israelis. As members of the court, Palestinians may be subject to counter-charges as well. Israel and the United States, neither of which is an ICC member, opposed the Palestinians\' efforts to join the body. But Palestinian Foreign Minister Riad al-Malki, speaking at Wednesday\'s ceremony, said it was a move toward greater justice. "As Palestine formally becomes a State Party to the Rome Statute today, the world is also a step closer to ending a long era of impunity and injustice," he said, according to an ICC news release. "Indeed, today brings us closer to our shared goals of justice and peace." Judge Kuniko Ozaki, a vice president of the ICC, said acceding to the treaty was just the first step for the Palestinians. "As the Rome Statute today enters into force for the State of Palestine, Palestine acquires all the rights as well as responsibilities that come with being a State Party to the Statute. These are substantive commitments, which cannot be taken lightly," she said. Rights group Human Rights Watch welcomed the development. "Governments seeking to penalize Palestine for joining the ICC should immediately end their pressure, and countries that support universal acceptance of the court\'s treaty should speak out to welcome its membership," said Balkees Jarrah, international justice counsel for the group. "What\'s objectionable is the attempts to undermine international justice, not Palestine\'s decision to join a treaty to which over 100 countries around the world are members." In January, when the preliminary ICC examination was opened, Israeli Prime Minister Benjamin Netanyahu described it as an outrage, saying the court was overstepping its boundaries. The United States also said it "strongly" disagreed with the court\'s decision. "As we have said repeatedly, we do not believe that Palestine is a state and therefore we do not believe that it is eligible to join the ICC," the State Department said in a statement. It urged the warring sides to resolve their differences through direct negotiations. "We will continue to oppose actions against Israel at the ICC as counterproductive to the cause of peace," it said. But the ICC begs to differ with the definition of a state for its purposes and refers to the territories as "Palestine." While a preliminary examination is not a formal investigation, it allows the court to review evidence and determine whether to investigate suspects on both sides. Prosecutor Fatou Bensouda said her office would "conduct its analysis in full independence and impartiality." The war between Israel and Hamas militants in Gaza last summer left more than 2,000 people dead. The inquiry will include alleged war crimes committed since June. The International Criminal Court was set up in 2002 to prosecute genocide, crimes against humanity and war crimes. CNN\'s Vasco Cotovio, Kareem Khadder and Faith Karimi contributed to this report.' + + # The below article tests that we don't add any hypotheses outside of the top n_beams + IRAN_ARTICLE = " (CNN)The United States and its negotiating partners reached a very strong framework agreement with Iran in Lausanne, Switzerland, on Thursday that limits Iran's nuclear program in such a way as to effectively block it from building a nuclear weapon. Expect pushback anyway, if the recent past is any harbinger. Just last month, in an attempt to head off such an agreement, House Speaker John Boehner invited Israeli Prime Minister Benjamin Netanyahu to preemptively blast it before Congress, and 47 senators sent a letter to the Iranian leadership warning them away from a deal. The debate that has already begun since the announcement of the new framework will likely result in more heat than light. It will not be helped by the gathering swirl of dubious assumptions and doubtful assertions. Let us address some of these: . The most misleading assertion, despite universal rejection by experts, is that the negotiations' objective at the outset was the total elimination of any nuclear program in Iran. That is the position of Netanyahu and his acolytes in the U.S. Congress. But that is not and never was the objective. If it had been, there would have been no Iranian team at the negotiating table. Rather, the objective has always been to structure an agreement or series of agreements so that Iran could not covertly develop a nuclear arsenal before the United States and its allies could respond. The new framework has exceeded expectations in achieving that goal. It would reduce Iran's low-enriched uranium stockpile, cut by two-thirds its number of installed centrifuges and implement a rigorous inspection regime. Another dubious assumption of opponents is that the Iranian nuclear program is a covert weapons program. Despite sharp accusations by some in the United States and its allies, Iran denies having such a program, and U.S. intelligence contends that Iran has not yet made the decision to build a nuclear weapon. Iran's continued cooperation with International Atomic Energy Agency inspections is further evidence on this point, and we'll know even more about Iran's program in the coming months and years because of the deal. In fact, the inspections provisions that are part of this agreement are designed to protect against any covert action by the Iranians. What's more, the rhetoric of some members of Congress has implied that the negotiations have been between only the United States and Iran (i.e., the 47 senators' letter warning that a deal might be killed by Congress or a future president). This of course is not the case. The talks were between Iran and the five permanent members of the U.N. Security Council (United States, United Kingdom, France, China and Russia) plus Germany, dubbed the P5+1. While the United States has played a leading role in the effort, it negotiated the terms alongside its partners. If the agreement reached by the P5+1 is rejected by Congress, it could result in an unraveling of the sanctions on Iran and threaten NATO cohesion in other areas. Another questionable assertion is that this agreement contains a sunset clause, after which Iran will be free to do as it pleases. Again, this is not the case. Some of the restrictions on Iran's nuclear activities, such as uranium enrichment, will be eased or eliminated over time, as long as 15 years. But most importantly, the framework agreement includes Iran's ratification of the Additional Protocol, which allows IAEA inspectors expanded access to nuclear sites both declared and nondeclared. This provision will be permanent. It does not sunset. Thus, going forward, if Iran decides to enrich uranium to weapons-grade levels, monitors will be able to detect such a move in a matter of days and alert the U.N. Security Council. Many in Congress have said that the agreement should be a formal treaty requiring the Senate to \"advise and consent.\" But the issue is not suited for a treaty. Treaties impose equivalent obligations on all signatories. For example, the New START treaty limits Russia and the United States to 1,550 deployed strategic warheads. But any agreement with Iran will not be so balanced. The restrictions and obligations in the final framework agreement will be imposed almost exclusively on Iran. The P5+1 are obligated only to ease and eventually remove most but not all economic sanctions, which were imposed as leverage to gain this final deal. Finally some insist that any agreement must address Iranian missile programs, human rights violations or support for Hamas or Hezbollah. As important as these issues are, and they must indeed be addressed, they are unrelated to the most important aim of a nuclear deal: preventing a nuclear Iran. To include them in the negotiations would be a poison pill. This agreement should be judged on its merits and on how it affects the security of our negotiating partners and allies, including Israel. Those judgments should be fact-based, not based on questionable assertions or dubious assumptions." + + ARTICLE_SUBWAY = ' New York (CNN)When Liana Barrientos was 23 years old, she got married in Westchester County, New York. A year later, she got married again in Westchester County, but to a different man and without divorcing her first husband. Only 18 days after that marriage, she got hitched yet again. Then, Barrientos declared "I do" five more times, sometimes only within two weeks of each other. In 2010, she married once more, this time in the Bronx. In an application for a marriage license, she stated it was her "first and only" marriage. Barrientos, now 39, is facing two criminal counts of "offering a false instrument for filing in the first degree," referring to her false statements on the 2010 marriage license application, according to court documents. Prosecutors said the marriages were part of an immigration scam. On Friday, she pleaded not guilty at State Supreme Court in the Bronx, according to her attorney, Christopher Wright, who declined to comment further. After leaving court, Barrientos was arrested and charged with theft of service and criminal trespass for allegedly sneaking into the New York subway through an emergency exit, said Detective Annette Markowski, a police spokeswoman. In total, Barrientos has been married 10 times, with nine of her marriages occurring between 1999 and 2002. All occurred either in Westchester County, Long Island, New Jersey or the Bronx. She is believed to still be married to four men, and at one time, she was married to eight men at once, prosecutors say. Prosecutors said the immigration scam involved some of her husbands, who filed for permanent residence status shortly after the marriages. Any divorces happened only after such filings were approved. It was unclear whether any of the men will be prosecuted. The case was referred to the Bronx District Attorney\'s Office by Immigration and Customs Enforcement and the Department of Homeland Security\'s Investigation Division. Seven of the men are from so-called "red-flagged" countries, including Egypt, Turkey, Georgia, Pakistan and Mali. Her eighth husband, Rashid Rajput, was deported in 2006 to his native Pakistan after an investigation by the Joint Terrorism Task Force. If convicted, Barrientos faces up to four years in prison. Her next court appearance is scheduled for May 18.' + + dct = tokenizer.batch_encode_plus( + [FRANCE_ARTICLE, SHORTER_ARTICLE, IRAN_ARTICLE, ARTICLE_SUBWAY], + max_length=1024, + padding="max_length", + truncation_strategy="only_first", + truncation=True, + return_tensors="np", + ) + + self.assertEqual(1024, dct["input_ids"].shape[1]) + hypotheses_batch = model.generate( + input_ids=dct["input_ids"], + attention_mask=dct["attention_mask"], + num_beams=2, + ).sequences + assert (hypotheses_batch[:, 1] == 0).all().item() + + EXPECTED = [ + "A French prosecutor says he is not aware of any video footage from on board the plane. Two German magazines claim to have found a cell phone video showing the crash. The publications say they watched the video, which was found by a source close to the investigation. All 150 on board the Germanwings flight were killed.", + "Palestinian Authority becomes 123rd member of the International Criminal Court. The move gives the court jurisdiction over alleged crimes in Palestinian territories. Israel and the United States opposed the Palestinians' efforts to join the body. But Palestinian Foreign Minister Riad al-Malki said it was a move toward greater justice.", + "U.S. and its negotiating partners reached a strong framework agreement with Iran. Peter Bergen: The debate that has already begun will likely result in more heat than light. Bergen: The most misleading assertion is that the negotiations' objective at the outset was the total elimination of any nuclear program.", + "Liana Barrientos, 39, has been married 10 times, sometimes within two weeks of each other. Prosecutors say the marriages were part of an immigration scam. She pleaded not guilty at State Supreme Court in the Bronx on Friday. If convicted, Barrientos faces up to four years in prison.", + ] + + generated_summaries = tokenizer.batch_decode( + hypotheses_batch.tolist(), clean_up_tokenization_spaces=True, skip_special_tokens=True + ) + assert generated_summaries == EXPECTED + + +class FlaxOPTStandaloneDecoderModelTester: + def __init__( + self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_attention_mask=True, + use_labels=False, + vocab_size=99, + hidden_size=16, + num_hidden_layers=2, + num_attention_heads=4, + intermediate_size=4, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=32, + eos_token_id=2, + pad_token_id=1, + bos_token_id=0, + initializer_range=0.02, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_attention_mask = use_attention_mask + self.use_labels = use_labels + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.eos_token_id = eos_token_id + self.pad_token_id = pad_token_id + self.bos_token_id = bos_token_id + self.initializer_range = initializer_range + + def prepare_config_and_inputs(self): + input_ids = jnp.clip(ids_tensor([self.batch_size, self.seq_length], self.vocab_size), 3, self.vocab_size) + + attention_mask = None + if self.use_attention_mask: + attention_mask = random_attention_mask([self.batch_size, self.seq_length]) + + config = OPTConfig( + vocab_size=self.vocab_size, + d_model=self.hidden_size, + encoder_layers=self.num_hidden_layers, + decoder_layers=self.num_hidden_layers, + encoder_attention_heads=self.num_attention_heads, + decoder_attention_heads=self.num_attention_heads, + encoder_ffn_dim=self.intermediate_size, + decoder_ffn_dim=self.intermediate_size, + dropout=self.hidden_dropout_prob, + attention_dropout=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + eos_token_id=self.eos_token_id, + bos_token_id=self.bos_token_id, + pad_token_id=self.pad_token_id, + initializer_range=self.initializer_range, + use_cache=False, + ) + + return config, input_ids, attention_mask + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + config, input_ids, attention_mask = config_and_inputs + inputs_dict = {"input_ids": input_ids, "attention_mask": attention_mask} + return config, inputs_dict + + def prepare_config_and_inputs_for_decoder(self): + config, input_ids, attention_mask = self.prepare_config_and_inputs() + + encoder_hidden_states = floats_tensor([self.batch_size, self.seq_length, self.hidden_size]) + encoder_attention_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + + return ( + config, + input_ids, + attention_mask, + encoder_hidden_states, + encoder_attention_mask, + ) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py new file mode 100644 index 0000000000000..c0c92fcd341ef --- /dev/null +++ b/tests/models/opt/test_modeling_opt.py @@ -0,0 +1,968 @@ +# coding=utf-8 +# Copyright 2021, The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Testing suite for the PyTorch OPT model. """ + + +import copy +import tempfile +import unittest + +import timeout_decorator # noqa + +from transformers import OPTConfig, is_torch_available +from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device +from transformers.utils import cached_property + +from ...generation.test_generation_utils import GenerationTesterMixin +from ...test_configuration_common import ConfigTester +from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor + + +if is_torch_available(): + import torch + + from transformers import ( + AutoModelForSequenceClassification, + OPTForCausalLM, + OPTForConditionalGeneration, + OPTForQuestionAnswering, + OPTForSequenceClassification, + OPTModel, + OPTTokenizer, + pipeline, + ) + from transformers.models.opt.modeling_opt import OPTDecoder, OPTEncoder, shift_tokens_right + + +def prepare_opt_inputs_dict( + config, + input_ids, + decoder_input_ids=None, + attention_mask=None, + decoder_attention_mask=None, + head_mask=None, + decoder_head_mask=None, + cross_attn_head_mask=None, +): + if attention_mask is None: + attention_mask = input_ids.ne(config.pad_token_id) + if decoder_attention_mask is None: + decoder_attention_mask = decoder_input_ids.ne(config.pad_token_id) + if head_mask is None: + head_mask = torch.ones(config.encoder_layers, config.encoder_attention_heads, device=torch_device) + if decoder_head_mask is None: + decoder_head_mask = torch.ones(config.decoder_layers, config.decoder_attention_heads, device=torch_device) + if cross_attn_head_mask is None: + cross_attn_head_mask = torch.ones(config.decoder_layers, config.decoder_attention_heads, device=torch_device) + return { + "input_ids": input_ids, + "decoder_input_ids": decoder_input_ids, + "attention_mask": attention_mask, + "decoder_attention_mask": attention_mask, + "head_mask": head_mask, + "decoder_head_mask": decoder_head_mask, + "cross_attn_head_mask": cross_attn_head_mask, + } + + +class OPTModelTester: + def __init__( + self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_labels=False, + vocab_size=99, + hidden_size=16, + num_hidden_layers=2, + num_attention_heads=4, + intermediate_size=4, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=20, + eos_token_id=2, + pad_token_id=1, + bos_token_id=0, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_labels = use_labels + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.eos_token_id = eos_token_id + self.pad_token_id = pad_token_id + self.bos_token_id = bos_token_id + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size).clamp( + 3, + ) + input_ids[:, -1] = self.eos_token_id # Eos Token + + decoder_input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + config = self.get_config() + inputs_dict = prepare_opt_inputs_dict(config, input_ids, decoder_input_ids) + return config, inputs_dict + + def get_config(self): + return OPTConfig( + vocab_size=self.vocab_size, + d_model=self.hidden_size, + encoder_layers=self.num_hidden_layers, + decoder_layers=self.num_hidden_layers, + encoder_attention_heads=self.num_attention_heads, + decoder_attention_heads=self.num_attention_heads, + encoder_ffn_dim=self.intermediate_size, + decoder_ffn_dim=self.intermediate_size, + dropout=self.hidden_dropout_prob, + attention_dropout=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + eos_token_id=self.eos_token_id, + bos_token_id=self.bos_token_id, + pad_token_id=self.pad_token_id, + ) + + def get_pipeline_config(self): + config = self.get_config() + config.max_position_embeddings = 100 + return config + + def prepare_config_and_inputs_for_common(self): + config, inputs_dict = self.prepare_config_and_inputs() + return config, inputs_dict + + def create_and_check_decoder_model_past_large_inputs(self, config, inputs_dict): + model = OPTModel(config=config).get_decoder().to(torch_device).eval() + input_ids = inputs_dict["input_ids"] + attention_mask = inputs_dict["attention_mask"] + head_mask = inputs_dict["head_mask"] + + # first forward pass + outputs = model(input_ids, attention_mask=attention_mask, head_mask=head_mask, use_cache=True) + + output, past_key_values = outputs.to_tuple() + + # create hypothetical multiple next token and extent to next_input_ids + next_tokens = ids_tensor((self.batch_size, 3), config.vocab_size) + next_attn_mask = ids_tensor((self.batch_size, 3), 2) + + # append to next input_ids and + next_input_ids = torch.cat([input_ids, next_tokens], dim=-1) + next_attention_mask = torch.cat([attention_mask, next_attn_mask], dim=-1) + + output_from_no_past = model(next_input_ids, attention_mask=next_attention_mask)["last_hidden_state"] + output_from_past = model(next_tokens, attention_mask=next_attention_mask, past_key_values=past_key_values)[ + "last_hidden_state" + ] + + # select random slice + random_slice_idx = ids_tensor((1,), output_from_past.shape[-1]).item() + output_from_no_past_slice = output_from_no_past[:, -3:, random_slice_idx].detach() + output_from_past_slice = output_from_past[:, :, random_slice_idx].detach() + + self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]) + + # test that outputs are equal for slice + self.parent.assertTrue(torch.allclose(output_from_past_slice, output_from_no_past_slice, atol=1e-3)) + + def check_encoder_decoder_model_standalone(self, config, inputs_dict): + model = OPTModel(config=config).to(torch_device).eval() + outputs = model(**inputs_dict) + + encoder_last_hidden_state = outputs.encoder_last_hidden_state + last_hidden_state = outputs.last_hidden_state + + with tempfile.TemporaryDirectory() as tmpdirname: + encoder = model.get_encoder() + encoder.save_pretrained(tmpdirname) + encoder = OPTEncoder.from_pretrained(tmpdirname).to(torch_device) + + encoder_last_hidden_state_2 = encoder(inputs_dict["input_ids"], attention_mask=inputs_dict["attention_mask"])[ + 0 + ] + + self.parent.assertTrue((encoder_last_hidden_state_2 - encoder_last_hidden_state).abs().max().item() < 1e-3) + + with tempfile.TemporaryDirectory() as tmpdirname: + decoder = model.get_decoder() + decoder.save_pretrained(tmpdirname) + decoder = OPTDecoder.from_pretrained(tmpdirname).to(torch_device) + + last_hidden_state_2 = decoder( + input_ids=inputs_dict["decoder_input_ids"], + attention_mask=inputs_dict["decoder_attention_mask"], + encoder_hidden_states=encoder_last_hidden_state, + encoder_attention_mask=inputs_dict["attention_mask"], + )[0] + + self.parent.assertTrue((last_hidden_state_2 - last_hidden_state).abs().max().item() < 1e-3) + + +@require_torch +class OPTHeadTests(unittest.TestCase): + vocab_size = 99 + + def _get_config_and_data(self): + input_ids = torch.tensor( + [ + [71, 82, 18, 33, 46, 91, 2], + [68, 34, 26, 58, 30, 82, 2], + [5, 97, 17, 39, 94, 40, 2], + [76, 83, 94, 25, 70, 78, 2], + [87, 59, 41, 35, 48, 66, 2], + [55, 13, 16, 58, 5, 2, 1], # note padding + [64, 27, 31, 51, 12, 75, 2], + [52, 64, 86, 17, 83, 39, 2], + [48, 61, 9, 24, 71, 82, 2], + [26, 1, 60, 48, 22, 13, 2], + [21, 5, 62, 28, 14, 76, 2], + [45, 98, 37, 86, 59, 48, 2], + [70, 70, 50, 9, 28, 0, 2], + ], + dtype=torch.long, + device=torch_device, + ) + + batch_size = input_ids.shape[0] + config = OPTConfig( + vocab_size=self.vocab_size, + d_model=24, + encoder_layers=2, + decoder_layers=2, + encoder_attention_heads=2, + decoder_attention_heads=2, + encoder_ffn_dim=32, + decoder_ffn_dim=32, + max_position_embeddings=48, + eos_token_id=2, + pad_token_id=1, + bos_token_id=0, + ) + return config, input_ids, batch_size + + def test_sequence_classification_forward(self): + config, input_ids, batch_size = self._get_config_and_data() + labels = _long_tensor([2] * batch_size).to(torch_device) + model = OPTForSequenceClassification(config) + model.to(torch_device) + outputs = model(input_ids=input_ids, decoder_input_ids=input_ids, labels=labels) + expected_shape = torch.Size((batch_size, config.num_labels)) + self.assertEqual(outputs["logits"].shape, expected_shape) + self.assertIsInstance(outputs["loss"].item(), float) + + def test_question_answering_forward(self): + config, input_ids, batch_size = self._get_config_and_data() + sequence_labels = ids_tensor([batch_size], 2).to(torch_device) + model = OPTForQuestionAnswering(config) + model.to(torch_device) + outputs = model( + input_ids=input_ids, + start_positions=sequence_labels, + end_positions=sequence_labels, + ) + + self.assertEqual(outputs["start_logits"].shape, input_ids.shape) + self.assertEqual(outputs["end_logits"].shape, input_ids.shape) + self.assertIsInstance(outputs["loss"].item(), float) + + @timeout_decorator.timeout(1) + def test_lm_forward(self): + config, input_ids, batch_size = self._get_config_and_data() + lm_labels = ids_tensor([batch_size, input_ids.shape[1]], self.vocab_size).to(torch_device) + lm_model = OPTForConditionalGeneration(config) + lm_model.to(torch_device) + outputs = lm_model(input_ids=input_ids, labels=lm_labels) + expected_shape = (batch_size, input_ids.shape[1], config.vocab_size) + self.assertEqual(outputs["logits"].shape, expected_shape) + self.assertIsInstance(outputs["loss"].item(), float) + + def test_lm_uneven_forward(self): + config = OPTConfig( + vocab_size=self.vocab_size, + d_model=14, + encoder_layers=2, + decoder_layers=2, + encoder_attention_heads=2, + decoder_attention_heads=2, + encoder_ffn_dim=8, + decoder_ffn_dim=8, + max_position_embeddings=48, + ) + lm_model = OPTForConditionalGeneration(config).to(torch_device) + context = torch.tensor( + [[71, 82, 18, 33, 46, 91, 2], [68, 34, 26, 58, 30, 2, 1]], device=torch_device, dtype=torch.long + ) + summary = torch.tensor([[82, 71, 82, 18, 2], [58, 68, 2, 1, 1]], device=torch_device, dtype=torch.long) + outputs = lm_model(input_ids=context, decoder_input_ids=summary, labels=summary) + expected_shape = (*summary.shape, config.vocab_size) + self.assertEqual(outputs["logits"].shape, expected_shape) + + def test_generate_beam_search(self): + input_ids = torch.tensor([[71, 82, 2], [68, 34, 2]], device=torch_device, dtype=torch.long) + config = OPTConfig( + vocab_size=self.vocab_size, + d_model=24, + encoder_layers=2, + decoder_layers=2, + encoder_attention_heads=2, + decoder_attention_heads=2, + encoder_ffn_dim=32, + decoder_ffn_dim=32, + max_position_embeddings=48, + eos_token_id=2, + pad_token_id=1, + bos_token_id=0, + ) + lm_model = OPTForConditionalGeneration(config).to(torch_device) + lm_model.eval() + + max_length = 5 + generated_ids = lm_model.generate( + input_ids.clone(), + do_sample=True, + num_return_sequences=1, + num_beams=2, + no_repeat_ngram_size=3, + max_length=max_length, + ) + self.assertEqual(generated_ids.shape, (input_ids.shape[0], max_length)) + + def test_shift_tokens_right(self): + input_ids = torch.tensor([[71, 82, 18, 33, 2, 1, 1], [68, 34, 26, 58, 30, 82, 2]], dtype=torch.long) + shifted = shift_tokens_right(input_ids, 1, 2) + n_pad_before = input_ids.eq(1).float().sum() + n_pad_after = shifted.eq(1).float().sum() + self.assertEqual(shifted.shape, input_ids.shape) + self.assertEqual(n_pad_after, n_pad_before - 1) + self.assertTrue(torch.eq(shifted[:, 0], 2).all()) + + @slow + def test_tokenization(self): + tokenizer = OPTTokenizer.from_pretrained("facebook/opt-large") + examples = [" Hello world", " DomDramg"] # need leading spaces for equality + fairseq_results = [ + torch.tensor([0, 20920, 232, 2]), + torch.tensor([0, 11349, 495, 4040, 571, 2]), + ] + for ex, desired_result in zip(examples, fairseq_results): + opt_toks = tokenizer.encode(ex, return_tensors="pt").squeeze() + assert_tensors_close(desired_result.long(), opt_toks, prefix=ex) + + def test_generate_fp16(self): + config, input_ids, batch_size = self._get_config_and_data() + attention_mask = input_ids.ne(1).to(torch_device) + model = OPTForConditionalGeneration(config).eval().to(torch_device) + if torch_device == "cuda": + model.half() + model.generate(input_ids, attention_mask=attention_mask) + model.generate(num_beams=4, do_sample=True, early_stopping=False, num_return_sequences=3) + + def test_dummy_inputs(self): + config, *_ = self._get_config_and_data() + model = OPTForConditionalGeneration(config).eval().to(torch_device) + model(**model.dummy_inputs) + + def test_resize_tokens_embeddings_more(self): + config, input_ids, _ = self._get_config_and_data() + + def _get_embs(m): + return (m.get_input_embeddings().weight.data.clone(), m.get_output_embeddings().weight.data.clone()) + + model = OPTForConditionalGeneration(config).eval().to(torch_device) + input, output = _get_embs(model) + self.assertTrue(torch.eq(input, output).all()) + new_vocab_size = 45 + model.resize_token_embeddings(new_vocab_size) + input_new, output_new = _get_embs(model) + self.assertEqual(input_new.shape, (new_vocab_size, config.d_model)) + self.assertEqual(output_new.shape, (new_vocab_size, config.d_model)) + self.assertTrue(torch.eq(input_new, output_new).all()) + + +@require_torch +class OPTModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase): + all_model_classes = ( + (OPTModel, OPTForConditionalGeneration, OPTForSequenceClassification, OPTForQuestionAnswering) + if is_torch_available() + else () + ) + all_generative_model_classes = (OPTForConditionalGeneration,) if is_torch_available() else () + is_encoder_decoder = True + test_pruning = False + test_missing_keys = False + + def setUp(self): + self.model_tester = OPTModelTester(self) + self.config_tester = ConfigTester(self, config_class=OPTConfig) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_save_load_strict(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs() + for model_class in self.all_model_classes: + model = model_class(config) + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname) + model2, info = model_class.from_pretrained(tmpdirname, output_loading_info=True) + self.assertEqual(info["missing_keys"], []) + + def test_decoder_model_past_with_large_inputs(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_decoder_model_past_large_inputs(*config_and_inputs) + + def test_encoder_decoder_model_standalone(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs_for_common() + self.model_tester.check_encoder_decoder_model_standalone(*config_and_inputs) + + # OPTForSequenceClassification does not support inputs_embeds + def test_inputs_embeds(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in (OPTModel, OPTForConditionalGeneration, OPTForQuestionAnswering): + model = model_class(config) + model.to(torch_device) + model.eval() + + inputs = copy.deepcopy(self._prepare_for_class(inputs_dict, model_class)) + + if not self.is_encoder_decoder: + input_ids = inputs["input_ids"] + del inputs["input_ids"] + else: + encoder_input_ids = inputs["input_ids"] + decoder_input_ids = inputs.get("decoder_input_ids", encoder_input_ids) + del inputs["input_ids"] + inputs.pop("decoder_input_ids", None) + + wte = model.get_input_embeddings() + if not self.is_encoder_decoder: + inputs["inputs_embeds"] = wte(input_ids) + else: + inputs["inputs_embeds"] = wte(encoder_input_ids) + inputs["decoder_inputs_embeds"] = wte(decoder_input_ids) + + with torch.no_grad(): + model(**inputs)[0] + + def test_generate_fp16(self): + config, input_dict = self.model_tester.prepare_config_and_inputs() + input_ids = input_dict["input_ids"] + attention_mask = input_ids.ne(1).to(torch_device) + model = OPTForConditionalGeneration(config).eval().to(torch_device) + if torch_device == "cuda": + model.half() + model.generate(input_ids, attention_mask=attention_mask) + model.generate(num_beams=4, do_sample=True, early_stopping=False, num_return_sequences=3) + + +def assert_tensors_close(a, b, atol=1e-12, prefix=""): + """If tensors have different shapes, different values or a and b are not both tensors, raise a nice Assertion error.""" + if a is None and b is None: + return True + try: + if torch.allclose(a, b, atol=atol): + return True + raise + except Exception: + pct_different = (torch.gt((a - b).abs(), atol)).float().mean().item() + if a.numel() > 100: + msg = f"tensor values are {pct_different:.1%} percent different." + else: + msg = f"{a} != {b}" + if prefix: + msg = prefix + ": " + msg + raise AssertionError(msg) + + +def _long_tensor(tok_lst): + return torch.tensor(tok_lst, dtype=torch.long, device=torch_device) + + +@require_torch +@slow +class FastIntegrationTests(unittest.TestCase): + """These tests are useful for debugging since they operate on a model with 1 encoder layer and 1 decoder layer.""" + + @cached_property + def tok(self): + return OPTTokenizer.from_pretrained("facebook/opt-large") + + @cached_property + def xsum_1_1_model(self): + return OPTForConditionalGeneration.from_pretrained("sshleifer/distilopt-xsum-1-1") + + def test_xsum_1_1_generation(self): + hf = self.xsum_1_1_model + tok = self.tok + ARTICLE = 'The Palestinian Authority officially became the 123rd member of the International Criminal Court on Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian territories. The formal accession was marked with a ceremony at The Hague, in the Netherlands, where the court is based. The Palestinians signed the ICC\'s founding Rome Statute in January, when they also accepted its jurisdiction over alleged crimes committed "in the occupied Palestinian territory, including East Jerusalem, since June 13, 2014." Later that month, the ICC opened a preliminary examination into the situation in Palestinian territories, paving the way for possible war crimes investigations against Israelis. As members of the court, Palestinians may be subject to counter-charges as well. Israel and the United States, neither of which is an ICC member, opposed the Palestinians\' efforts to join the body. But Palestinian Foreign Minister Riad al-Malki, speaking at Wednesday\'s ceremony, said it was a move toward greater justice. "As Palestine formally becomes a State Party to the Rome Statute today, the world is also a step closer to ending a long era of impunity and injustice," he said, according to an ICC news release. "Indeed, today brings us closer to our shared goals of justice and peace." Judge Kuniko Ozaki, a vice president of the ICC, said acceding to the treaty was just the first step for the Palestinians. "As the Rome Statute today enters into force for the State of Palestine, Palestine acquires all the rights as well as responsibilities that come with being a State Party to the Statute. These are substantive commitments, which cannot be taken lightly," she said. Rights group Human Rights Watch welcomed the development. "Governments seeking to penalize Palestine for joining the ICC should immediately end their pressure, and countries that support universal acceptance of the court\'s treaty should speak out to welcome its membership," said Balkees Jarrah, international justice counsel for the group. "What\'s objectionable is the attempts to undermine international justice, not Palestine\'s decision to join a treaty to which over 100 countries around the world are members." In January, when the preliminary ICC examination was opened, Israeli Prime Minister Benjamin Netanyahu described it as an outrage, saying the court was overstepping its boundaries. The United States also said it "strongly" disagreed with the court\'s decision. "As we have said repeatedly, we do not believe that Palestine is a state and therefore we do not believe that it is eligible to join the ICC," the State Department said in a statement. It urged the warring sides to resolve their differences through direct negotiations. "We will continue to oppose actions against Israel at the ICC as counterproductive to the cause of peace," it said. But the ICC begs to differ with the definition of a state for its purposes and refers to the territories as "Palestine." While a preliminary examination is not a formal investigation, it allows the court to review evidence and determine whether to investigate suspects on both sides. Prosecutor Fatou Bensouda said her office would "conduct its analysis in full independence and impartiality." The war between Israel and Hamas militants in Gaza last summer left more than 2,000 people dead. The inquiry will include alleged war crimes committed since June. The International Criminal Court was set up in 2002 to prosecute genocide, crimes against humanity and war crimes.' + EXPECTED = " The International Criminal Court (ICC) has announced that it has been announced by the International Criminal court." + + dct = tok(ARTICLE, return_tensors="pt") + generated_ids = hf.generate(**dct, num_beams=4) + result = tok.batch_decode(generated_ids, skip_special_tokens=True)[0] + assert EXPECTED == result + + def test_xsum_1_1_batch_generation(self): + # test batch + + batch = self.tok( + [ + 'The Palestinian Authority officially became the 123rd member of the International Criminal Court on Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian territories. The formal accession was marked with a ceremony at The Hague, in the Netherlands, where the court is based. The Palestinians signed the ICC\'s founding Rome Statute in January, when they also accepted its jurisdiction over alleged crimes committed "in the occupied Palestinian territory, including East Jerusalem, since June 13, 2014." Later that month, the ICC opened a preliminary examination into the situation in Palestinian territories, paving the way for possible war crimes investigations against Israelis. As members of the court, Palestinians may be subject to counter-charges as well. Israel and the United States, neither of which is an ICC member, opposed the Palestinians\' efforts to join the body. But Palestinian Foreign Minister Riad al-Malki, speaking at Wednesday\'s ceremony, said it was a move toward greater justice. "As Palestine formally becomes a State Party to the Rome Statute today, the world is also a step closer to ending a long era of impunity and injustice," he said, according to an ICC news release. "Indeed, today brings us closer to our shared goals of justice and peace." Judge Kuniko Ozaki, a vice president of the ICC, said acceding to the treaty was just the first step for the Palestinians. "As the Rome Statute today enters into force for the State of Palestine, Palestine acquires all the rights as well as responsibilities that come with being a State Party to the Statute. These are substantive commitments, which cannot be taken lightly," she said. Rights group Human Rights Watch welcomed the development. "Governments seeking to penalize Palestine for joining the ICC should immediately end their pressure, and countries that support universal acceptance of the court\'s treaty should speak out to welcome its membership," said Balkees Jarrah, international justice counsel for the group. "What\'s objectionable is the attempts to undermine international justice, not Palestine\'s decision to join a treaty to which over 100 countries around the world are members." In January, when the preliminary ICC examination was opened, Israeli Prime Minister Benjamin Netanyahu described it as an outrage, saying the court was overstepping its boundaries. The United States also said it "strongly" disagreed with the court\'s decision. "As we have said repeatedly, we do not believe that Palestine is a state and therefore we do not believe that it is eligible to join the ICC," the State Department said in a statement. It urged the warring sides to resolve their differences through direct negotiations. "We will continue to oppose actions against Israel at the ICC as counterproductive to the cause of peace," it said. But the ICC begs to differ with the definition of a state for its purposes and refers to the territories as "Palestine." While a preliminary examination is not a formal investigation, it allows the court to review evidence and determine whether to investigate suspects on both sides. Prosecutor Fatou Bensouda said her office would "conduct its analysis in full independence and impartiality." The war between Israel and Hamas militants in Gaza last summer left more than 2,000 people dead. The inquiry will include alleged war crimes committed since June. The International Criminal Court was set up in 2002 to prosecute genocide, crimes against humanity and war crimes.', + 'The French prosecutor leading an investigation into the crash of Germanwings Flight 9525 insisted Wednesday that he was not aware of any video footage from on board the plane. Marseille prosecutor Brice Robin told CNN that "so far no videos were used in the crash investigation." He added, "A person who has such a video needs to immediately give it to the investigators." Robin\'s comments follow claims by two magazines, German daily Bild and French Paris Match, of a cell phone video showing the harrowing final seconds from on board Germanwings Flight 9525 as it crashed into the French Alps. All 150 on board were killed. Paris Match and Bild reported that the video was recovered from a phone at the wreckage site. The two publications described the supposed video, but did not post it on their websites. The publications said that they watched the video, which was found by a source close to the investigation. "One can hear cries of \'My God\' in several languages," Paris Match reported. "Metallic banging can also be heard more than three times, perhaps of the pilot trying to open the cockpit door with a heavy object. Towards the end, after a heavy shake, stronger than the others, the screaming intensifies. Then nothing." "It is a very disturbing scene," said Julian Reichelt, editor-in-chief of Bild online. An official with France\'s accident investigation agency, the BEA, said the agency is not aware of any such video. Lt. Col. Jean-Marc Menichini, a French Gendarmerie spokesman in charge of communications on rescue efforts around the Germanwings crash site, told CNN that the reports were "completely wrong" and "unwarranted." Cell phones have been collected at the site, he said, but that they "hadn\'t been exploited yet." Menichini said he believed the cell phones would need to be sent to the Criminal Research Institute in Rosny sous-Bois, near Paris, in order to be analyzed by specialized technicians working hand-in-hand with investigators. But none of the cell phones found so far have been sent to the institute, Menichini said. Asked whether staff involved in the search could have leaked a memory card to the media, Menichini answered with a categorical "no." Reichelt told "Erin Burnett: Outfront" that he had watched the video and stood by the report, saying Bild and Paris Match are "very confident" that the clip is real. He noted that investigators only revealed they\'d recovered cell phones from the crash site after Bild and Paris Match published their reports. "That is something we did not know before. ... Overall we can say many things of the investigation weren\'t revealed by the investigation at the beginning," he said. What was mental state of Germanwings co-pilot? German airline Lufthansa confirmed Tuesday that co-pilot Andreas Lubitz had battled depression years before he took the controls of Germanwings Flight 9525, which he\'s accused of deliberately crashing last week in the French Alps. Lubitz told his Lufthansa flight training school in 2009 that he had a "previous episode of severe depression," the airline said Tuesday. Email correspondence between Lubitz and the school discovered in an internal investigation, Lufthansa said, included medical documents he submitted in connection with resuming his flight training. The announcement indicates that Lufthansa, the parent company of Germanwings, knew of Lubitz\'s battle with depression, allowed him to continue training and ultimately put him in the cockpit. Lufthansa, whose CEO Carsten Spohr previously said Lubitz was 100% fit to fly, described its statement Tuesday as a "swift and seamless clarification" and said it was sharing the information and documents -- including training and medical records -- with public prosecutors. Spohr traveled to the crash site Wednesday, where recovery teams have been working for the past week to recover human remains and plane debris scattered across a steep mountainside. He saw the crisis center set up in Seyne-les-Alpes, laid a wreath in the village of Le Vernet, closer to the crash site, where grieving families have left flowers at a simple stone memorial. Menichini told CNN late Tuesday that no visible human remains were left at the site but recovery teams would keep searching. French President Francois Hollande, speaking Tuesday, said that it should be possible to identify all the victims using DNA analysis by the end of the week, sooner than authorities had previously suggested. In the meantime, the recovery of the victims\' personal belongings will start Wednesday, Menichini said. Among those personal belongings could be more cell phones belonging to the 144 passengers and six crew on board. Check out the latest from our correspondents . The details about Lubitz\'s correspondence with the flight school during his training were among several developments as investigators continued to delve into what caused the crash and Lubitz\'s possible motive for downing the jet. A Lufthansa spokesperson told CNN on Tuesday that Lubitz had a valid medical certificate, had passed all his examinations and "held all the licenses required." Earlier, a spokesman for the prosecutor\'s office in Dusseldorf, Christoph Kumpa, said medical records reveal Lubitz suffered from suicidal tendencies at some point before his aviation career and underwent psychotherapy before he got his pilot\'s license. Kumpa emphasized there\'s no evidence suggesting Lubitz was suicidal or acting aggressively before the crash. Investigators are looking into whether Lubitz feared his medical condition would cause him to lose his pilot\'s license, a European government official briefed on the investigation told CNN on Tuesday. While flying was "a big part of his life," the source said, it\'s only one theory being considered. Another source, a law enforcement official briefed on the investigation, also told CNN that authorities believe the primary motive for Lubitz to bring down the plane was that he feared he would not be allowed to fly because of his medical problems. Lubitz\'s girlfriend told investigators he had seen an eye doctor and a neuropsychologist, both of whom deemed him unfit to work recently and concluded he had psychological issues, the European government official said. But no matter what details emerge about his previous mental health struggles, there\'s more to the story, said Brian Russell, a forensic psychologist. "Psychology can explain why somebody would turn rage inward on themselves about the fact that maybe they weren\'t going to keep doing their job and they\'re upset about that and so they\'re suicidal," he said. "But there is no mental illness that explains why somebody then feels entitled to also take that rage and turn it outward on 149 other people who had nothing to do with the person\'s problems." Germanwings crash compensation: What we know . Who was the captain of Germanwings Flight 9525? CNN\'s Margot Haddad reported from Marseille and Pamela Brown from Dusseldorf, while Laura Smith-Spark wrote from London. CNN\'s Frederik Pleitgen, Pamela Boykoff, Antonia Mortensen, Sandrine Amiel and Anna-Maja Rappard contributed to this report.', + ], + return_tensors="pt", + padding="longest", + truncation=True, + ) + generated_ids = self.xsum_1_1_model.generate(**batch, num_beams=4) + result = self.tok.batch_decode(generated_ids, skip_special_tokens=True) + assert ( + result[0] + == " The International Criminal Court (ICC) has announced that it has been announced by the International Criminal court." + ) + assert ( + result[1] + == " An investigation into the crash that killed at least 10 people in the French capital has been released by the French police investigating the crash." + ) + + def test_encoder_equiv(self): + # test batch + + batch = self.tok( + [ + 'The Palestinian Authority officially became the 123rd member of the International Criminal Court on Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian territories. The formal accession was marked with a ceremony at The Hague, in the Netherlands, where the court is based. The Palestinians signed the ICC\'s founding Rome Statute in January, when they also accepted its jurisdiction over alleged crimes committed "in the occupied Palestinian territory, including East Jerusalem, since June 13, 2014." Later that month, the ICC opened a preliminary examination into the situation in Palestinian territories, paving the way for possible war crimes investigations against Israelis. As members of the court, Palestinians may be subject to counter-charges as well. Israel and the United States, neither of which is an ICC member, opposed the Palestinians\' efforts to join the body. But Palestinian Foreign Minister Riad al-Malki, speaking at Wednesday\'s ceremony, said it was a move toward greater justice. "As Palestine formally becomes a State Party to the Rome Statute today, the world is also a step closer to ending a long era of impunity and injustice," he said, according to an ICC news release. "Indeed, today brings us closer to our shared goals of justice and peace." Judge Kuniko Ozaki, a vice president of the ICC, said acceding to the treaty was just the first step for the Palestinians. "As the Rome Statute today enters into force for the State of Palestine, Palestine acquires all the rights as well as responsibilities that come with being a State Party to the Statute. These are substantive commitments, which cannot be taken lightly," she said. Rights group Human Rights Watch welcomed the development. "Governments seeking to penalize Palestine for joining the ICC should immediately end their pressure, and countries that support universal acceptance of the court\'s treaty should speak out to welcome its membership," said Balkees Jarrah, international justice counsel for the group. "What\'s objectionable is the attempts to undermine international justice, not Palestine\'s decision to join a treaty to which over 100 countries around the world are members." In January, when the preliminary ICC examination was opened, Israeli Prime Minister Benjamin Netanyahu described it as an outrage, saying the court was overstepping its boundaries. The United States also said it "strongly" disagreed with the court\'s decision. "As we have said repeatedly, we do not believe that Palestine is a state and therefore we do not believe that it is eligible to join the ICC," the State Department said in a statement. It urged the warring sides to resolve their differences through direct negotiations. "We will continue to oppose actions against Israel at the ICC as counterproductive to the cause of peace," it said. But the ICC begs to differ with the definition of a state for its purposes and refers to the territories as "Palestine." While a preliminary examination is not a formal investigation, it allows the court to review evidence and determine whether to investigate suspects on both sides. Prosecutor Fatou Bensouda said her office would "conduct its analysis in full independence and impartiality." The war between Israel and Hamas militants in Gaza last summer left more than 2,000 people dead. The inquiry will include alleged war crimes committed since June. The International Criminal Court was set up in 2002 to prosecute genocide, crimes against humanity and war crimes.', + 'The French prosecutor leading an investigation into the crash of Germanwings Flight 9525 insisted Wednesday that he was not aware of any video footage from on board the plane. Marseille prosecutor Brice Robin told CNN that "so far no videos were used in the crash investigation." He added, "A person who has such a video needs to immediately give it to the investigators." Robin\'s comments follow claims by two magazines, German daily Bild and French Paris Match, of a cell phone video showing the harrowing final seconds from on board Germanwings Flight 9525 as it crashed into the French Alps. All 150 on board were killed. Paris Match and Bild reported that the video was recovered from a phone at the wreckage site. The two publications described the supposed video, but did not post it on their websites. The publications said that they watched the video, which was found by a source close to the investigation. "One can hear cries of \'My God\' in several languages," Paris Match reported. "Metallic banging can also be heard more than three times, perhaps of the pilot trying to open the cockpit door with a heavy object. Towards the end, after a heavy shake, stronger than the others, the screaming intensifies. Then nothing." "It is a very disturbing scene," said Julian Reichelt, editor-in-chief of Bild online. An official with France\'s accident investigation agency, the BEA, said the agency is not aware of any such video. Lt. Col. Jean-Marc Menichini, a French Gendarmerie spokesman in charge of communications on rescue efforts around the Germanwings crash site, told CNN that the reports were "completely wrong" and "unwarranted." Cell phones have been collected at the site, he said, but that they "hadn\'t been exploited yet." Menichini said he believed the cell phones would need to be sent to the Criminal Research Institute in Rosny sous-Bois, near Paris, in order to be analyzed by specialized technicians working hand-in-hand with investigators. But none of the cell phones found so far have been sent to the institute, Menichini said. Asked whether staff involved in the search could have leaked a memory card to the media, Menichini answered with a categorical "no." Reichelt told "Erin Burnett: Outfront" that he had watched the video and stood by the report, saying Bild and Paris Match are "very confident" that the clip is real. He noted that investigators only revealed they\'d recovered cell phones from the crash site after Bild and Paris Match published their reports. "That is something we did not know before. ... Overall we can say many things of the investigation weren\'t revealed by the investigation at the beginning," he said. What was mental state of Germanwings co-pilot? German airline Lufthansa confirmed Tuesday that co-pilot Andreas Lubitz had battled depression years before he took the controls of Germanwings Flight 9525, which he\'s accused of deliberately crashing last week in the French Alps. Lubitz told his Lufthansa flight training school in 2009 that he had a "previous episode of severe depression," the airline said Tuesday. Email correspondence between Lubitz and the school discovered in an internal investigation, Lufthansa said, included medical documents he submitted in connection with resuming his flight training. The announcement indicates that Lufthansa, the parent company of Germanwings, knew of Lubitz\'s battle with depression, allowed him to continue training and ultimately put him in the cockpit. Lufthansa, whose CEO Carsten Spohr previously said Lubitz was 100% fit to fly, described its statement Tuesday as a "swift and seamless clarification" and said it was sharing the information and documents -- including training and medical records -- with public prosecutors. Spohr traveled to the crash site Wednesday, where recovery teams have been working for the past week to recover human remains and plane debris scattered across a steep mountainside. He saw the crisis center set up in Seyne-les-Alpes, laid a wreath in the village of Le Vernet, closer to the crash site, where grieving families have left flowers at a simple stone memorial. Menichini told CNN late Tuesday that no visible human remains were left at the site but recovery teams would keep searching. French President Francois Hollande, speaking Tuesday, said that it should be possible to identify all the victims using DNA analysis by the end of the week, sooner than authorities had previously suggested. In the meantime, the recovery of the victims\' personal belongings will start Wednesday, Menichini said. Among those personal belongings could be more cell phones belonging to the 144 passengers and six crew on board. Check out the latest from our correspondents . The details about Lubitz\'s correspondence with the flight school during his training were among several developments as investigators continued to delve into what caused the crash and Lubitz\'s possible motive for downing the jet. A Lufthansa spokesperson told CNN on Tuesday that Lubitz had a valid medical certificate, had passed all his examinations and "held all the licenses required." Earlier, a spokesman for the prosecutor\'s office in Dusseldorf, Christoph Kumpa, said medical records reveal Lubitz suffered from suicidal tendencies at some point before his aviation career and underwent psychotherapy before he got his pilot\'s license. Kumpa emphasized there\'s no evidence suggesting Lubitz was suicidal or acting aggressively before the crash. Investigators are looking into whether Lubitz feared his medical condition would cause him to lose his pilot\'s license, a European government official briefed on the investigation told CNN on Tuesday. While flying was "a big part of his life," the source said, it\'s only one theory being considered. Another source, a law enforcement official briefed on the investigation, also told CNN that authorities believe the primary motive for Lubitz to bring down the plane was that he feared he would not be allowed to fly because of his medical problems. Lubitz\'s girlfriend told investigators he had seen an eye doctor and a neuropsychologist, both of whom deemed him unfit to work recently and concluded he had psychological issues, the European government official said. But no matter what details emerge about his previous mental health struggles, there\'s more to the story, said Brian Russell, a forensic psychologist. "Psychology can explain why somebody would turn rage inward on themselves about the fact that maybe they weren\'t going to keep doing their job and they\'re upset about that and so they\'re suicidal," he said. "But there is no mental illness that explains why somebody then feels entitled to also take that rage and turn it outward on 149 other people who had nothing to do with the person\'s problems." Germanwings crash compensation: What we know . Who was the captain of Germanwings Flight 9525? CNN\'s Margot Haddad reported from Marseille and Pamela Brown from Dusseldorf, while Laura Smith-Spark wrote from London. CNN\'s Frederik Pleitgen, Pamela Boykoff, Antonia Mortensen, Sandrine Amiel and Anna-Maja Rappard contributed to this report.', + ], + return_tensors="pt", + padding="longest", + truncation=True, + ) + features = self.xsum_1_1_model.get_encoder()(**batch).last_hidden_state + expected = [[-0.0828, -0.0251, -0.0674], [0.1277, 0.3311, -0.0255], [0.2613, -0.0840, -0.2763]] + assert_tensors_close(features[0, :3, :3], torch.tensor(expected), atol=1e-3) + + +@require_torch +@require_sentencepiece +@require_tokenizers +class OPTModelIntegrationTests(unittest.TestCase): + @cached_property + def default_tokenizer(self): + return OPTTokenizer.from_pretrained("facebook/opt-large") + + @slow + def test_inference_no_head(self): + model = OPTModel.from_pretrained("facebook/opt-large").to(torch_device) + input_ids = _long_tensor([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]) + attention_mask = input_ids.ne(model.config.pad_token_id) + with torch.no_grad(): + output = model(input_ids=input_ids, attention_mask=attention_mask).last_hidden_state + expected_shape = torch.Size((1, 11, 1024)) + self.assertEqual(output.shape, expected_shape) + expected_slice = torch.tensor( + [[0.7144, 0.8143, -1.2813], [0.7144, 0.8143, -1.2813], [-0.0467, 2.5911, -2.1845]], device=torch_device + ) + self.assertTrue(torch.allclose(output[:, :3, :3], expected_slice, atol=1e-3)) + + @slow + def test_base_mask_filling(self): + pbase = pipeline(task="fill-mask", model="") + src_text = [" I went to the ."] + results = [x["token_str"] for x in pbase(src_text)] + assert " bathroom" in results + + @slow + def test_large_mask_filling(self): + plarge = pipeline(task="fill-mask", model="facebook/opt-large") + src_text = [" I went to the ."] + results = [x["token_str"] for x in plarge(src_text)] + expected_results = [" bathroom", " gym", " wrong", " movies", " hospital"] + self.assertListEqual(results, expected_results) + + @slow + def test_mnli_inference(self): + example_b = [0, 31414, 232, 328, 740, 1140, 69, 46078, 1588, 2, 1] + input_ids = _long_tensor([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2], example_b]) + + model = AutoModelForSequenceClassification.from_pretrained("facebook/opt-large-mnli").to( + torch_device + ) # eval called in from_pre + attention_mask = input_ids.ne(model.config.pad_token_id) + # Test that model hasn't changed + with torch.no_grad(): + outputs = model(input_ids=input_ids, attention_mask=attention_mask) + + batched_logits = outputs.logits + expected_shape = torch.Size((2, 3)) + self.assertEqual(batched_logits.shape, expected_shape) + expected_slice = torch.tensor([[0.1907, 1.4342, -1.0289]], device=torch_device) + logits_arr = batched_logits[0].detach() + + # Test that padding does not change results + input_ids_no_pad = _long_tensor([example_b[:-1]]) + attention_mask_no_pad = input_ids_no_pad.ne(model.config.pad_token_id) + + with torch.no_grad(): + logits2 = model(input_ids=input_ids_no_pad, attention_mask=attention_mask_no_pad).logits.squeeze() + assert_tensors_close(batched_logits[1], logits2, atol=1e-3) + assert_tensors_close(expected_slice, logits_arr, atol=1e-3) + + @slow + def test_xsum_summarization_same_as_fairseq(self): + model = OPTForConditionalGeneration.from_pretrained("facebook/opt-large-xsum").to(torch_device) + tok = self.default_tokenizer + + PGE_ARTICLE = """ PG&E stated it scheduled the blackouts in response to forecasts for high winds amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow.""" + + EXPECTED_SUMMARY = "California's largest power company has begun shutting off electricity to thousands of customers in the state." + dct = tok.batch_encode_plus( + [PGE_ARTICLE], + max_length=1024, + padding="max_length", + truncation=True, + return_tensors="pt", + ).to(torch_device) + + hypotheses_batch = model.generate( + input_ids=dct["input_ids"], + attention_mask=dct["attention_mask"], + num_beams=2, + max_length=62, + min_length=11, + length_penalty=1.0, + no_repeat_ngram_size=3, + early_stopping=True, + decoder_start_token_id=model.config.eos_token_id, + ) + + decoded = tok.batch_decode( + hypotheses_batch, + skip_special_tokens=True, + ) + self.assertEqual(EXPECTED_SUMMARY, decoded[0]) + + def test_xsum_config_generation_params(self): + config = OPTConfig.from_pretrained("facebook/opt-large-xsum") + expected_params = dict(num_beams=6, do_sample=False, early_stopping=True, length_penalty=1.0) + config_params = {k: getattr(config, k, "MISSING") for k, v in expected_params.items()} + self.assertDictEqual(expected_params, config_params) + + @slow + def test_cnn_summarization_same_as_fairseq(self): + hf = OPTForConditionalGeneration.from_pretrained("facebook/opt-large-cnn").to(torch_device) + tok = OPTTokenizer.from_pretrained("facebook/opt-large") + + FRANCE_ARTICLE = ' Marseille, France (CNN)The French prosecutor leading an investigation into the crash of Germanwings Flight 9525 insisted Wednesday that he was not aware of any video footage from on board the plane. Marseille prosecutor Brice Robin told CNN that "so far no videos were used in the crash investigation." He added, "A person who has such a video needs to immediately give it to the investigators." Robin\'s comments follow claims by two magazines, German daily Bild and French Paris Match, of a cell phone video showing the harrowing final seconds from on board Germanwings Flight 9525 as it crashed into the French Alps. All 150 on board were killed. Paris Match and Bild reported that the video was recovered from a phone at the wreckage site. The two publications described the supposed video, but did not post it on their websites. The publications said that they watched the video, which was found by a source close to the investigation. "One can hear cries of \'My God\' in several languages," Paris Match reported. "Metallic banging can also be heard more than three times, perhaps of the pilot trying to open the cockpit door with a heavy object. Towards the end, after a heavy shake, stronger than the others, the screaming intensifies. Then nothing." "It is a very disturbing scene," said Julian Reichelt, editor-in-chief of Bild online. An official with France\'s accident investigation agency, the BEA, said the agency is not aware of any such video. Lt. Col. Jean-Marc Menichini, a French Gendarmerie spokesman in charge of communications on rescue efforts around the Germanwings crash site, told CNN that the reports were "completely wrong" and "unwarranted." Cell phones have been collected at the site, he said, but that they "hadn\'t been exploited yet." Menichini said he believed the cell phones would need to be sent to the Criminal Research Institute in Rosny sous-Bois, near Paris, in order to be analyzed by specialized technicians working hand-in-hand with investigators. But none of the cell phones found so far have been sent to the institute, Menichini said. Asked whether staff involved in the search could have leaked a memory card to the media, Menichini answered with a categorical "no." Reichelt told "Erin Burnett: Outfront" that he had watched the video and stood by the report, saying Bild and Paris Match are "very confident" that the clip is real. He noted that investigators only revealed they\'d recovered cell phones from the crash site after Bild and Paris Match published their reports. "That is something we did not know before. ... Overall we can say many things of the investigation weren\'t revealed by the investigation at the beginning," he said. What was mental state of Germanwings co-pilot? German airline Lufthansa confirmed Tuesday that co-pilot Andreas Lubitz had battled depression years before he took the controls of Germanwings Flight 9525, which he\'s accused of deliberately crashing last week in the French Alps. Lubitz told his Lufthansa flight training school in 2009 that he had a "previous episode of severe depression," the airline said Tuesday. Email correspondence between Lubitz and the school discovered in an internal investigation, Lufthansa said, included medical documents he submitted in connection with resuming his flight training. The announcement indicates that Lufthansa, the parent company of Germanwings, knew of Lubitz\'s battle with depression, allowed him to continue training and ultimately put him in the cockpit. Lufthansa, whose CEO Carsten Spohr previously said Lubitz was 100% fit to fly, described its statement Tuesday as a "swift and seamless clarification" and said it was sharing the information and documents -- including training and medical records -- with public prosecutors. Spohr traveled to the crash site Wednesday, where recovery teams have been working for the past week to recover human remains and plane debris scattered across a steep mountainside. He saw the crisis center set up in Seyne-les-Alpes, laid a wreath in the village of Le Vernet, closer to the crash site, where grieving families have left flowers at a simple stone memorial. Menichini told CNN late Tuesday that no visible human remains were left at the site but recovery teams would keep searching. French President Francois Hollande, speaking Tuesday, said that it should be possible to identify all the victims using DNA analysis by the end of the week, sooner than authorities had previously suggested. In the meantime, the recovery of the victims\' personal belongings will start Wednesday, Menichini said. Among those personal belongings could be more cell phones belonging to the 144 passengers and six crew on board. Check out the latest from our correspondents . The details about Lubitz\'s correspondence with the flight school during his training were among several developments as investigators continued to delve into what caused the crash and Lubitz\'s possible motive for downing the jet. A Lufthansa spokesperson told CNN on Tuesday that Lubitz had a valid medical certificate, had passed all his examinations and "held all the licenses required." Earlier, a spokesman for the prosecutor\'s office in Dusseldorf, Christoph Kumpa, said medical records reveal Lubitz suffered from suicidal tendencies at some point before his aviation career and underwent psychotherapy before he got his pilot\'s license. Kumpa emphasized there\'s no evidence suggesting Lubitz was suicidal or acting aggressively before the crash. Investigators are looking into whether Lubitz feared his medical condition would cause him to lose his pilot\'s license, a European government official briefed on the investigation told CNN on Tuesday. While flying was "a big part of his life," the source said, it\'s only one theory being considered. Another source, a law enforcement official briefed on the investigation, also told CNN that authorities believe the primary motive for Lubitz to bring down the plane was that he feared he would not be allowed to fly because of his medical problems. Lubitz\'s girlfriend told investigators he had seen an eye doctor and a neuropsychologist, both of whom deemed him unfit to work recently and concluded he had psychological issues, the European government official said. But no matter what details emerge about his previous mental health struggles, there\'s more to the story, said Brian Russell, a forensic psychologist. "Psychology can explain why somebody would turn rage inward on themselves about the fact that maybe they weren\'t going to keep doing their job and they\'re upset about that and so they\'re suicidal," he said. "But there is no mental illness that explains why somebody then feels entitled to also take that rage and turn it outward on 149 other people who had nothing to do with the person\'s problems." Germanwings crash compensation: What we know . Who was the captain of Germanwings Flight 9525? CNN\'s Margot Haddad reported from Marseille and Pamela Brown from Dusseldorf, while Laura Smith-Spark wrote from London. CNN\'s Frederik Pleitgen, Pamela Boykoff, Antonia Mortensen, Sandrine Amiel and Anna-Maja Rappard contributed to this report.' # @noq + + SHORTER_ARTICLE = ' (CNN)The Palestinian Authority officially became the 123rd member of the International Criminal Court on Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian territories. The formal accession was marked with a ceremony at The Hague, in the Netherlands, where the court is based. The Palestinians signed the ICC\'s founding Rome Statute in January, when they also accepted its jurisdiction over alleged crimes committed "in the occupied Palestinian territory, including East Jerusalem, since June 13, 2014." Later that month, the ICC opened a preliminary examination into the situation in Palestinian territories, paving the way for possible war crimes investigations against Israelis. As members of the court, Palestinians may be subject to counter-charges as well. Israel and the United States, neither of which is an ICC member, opposed the Palestinians\' efforts to join the body. But Palestinian Foreign Minister Riad al-Malki, speaking at Wednesday\'s ceremony, said it was a move toward greater justice. "As Palestine formally becomes a State Party to the Rome Statute today, the world is also a step closer to ending a long era of impunity and injustice," he said, according to an ICC news release. "Indeed, today brings us closer to our shared goals of justice and peace." Judge Kuniko Ozaki, a vice president of the ICC, said acceding to the treaty was just the first step for the Palestinians. "As the Rome Statute today enters into force for the State of Palestine, Palestine acquires all the rights as well as responsibilities that come with being a State Party to the Statute. These are substantive commitments, which cannot be taken lightly," she said. Rights group Human Rights Watch welcomed the development. "Governments seeking to penalize Palestine for joining the ICC should immediately end their pressure, and countries that support universal acceptance of the court\'s treaty should speak out to welcome its membership," said Balkees Jarrah, international justice counsel for the group. "What\'s objectionable is the attempts to undermine international justice, not Palestine\'s decision to join a treaty to which over 100 countries around the world are members." In January, when the preliminary ICC examination was opened, Israeli Prime Minister Benjamin Netanyahu described it as an outrage, saying the court was overstepping its boundaries. The United States also said it "strongly" disagreed with the court\'s decision. "As we have said repeatedly, we do not believe that Palestine is a state and therefore we do not believe that it is eligible to join the ICC," the State Department said in a statement. It urged the warring sides to resolve their differences through direct negotiations. "We will continue to oppose actions against Israel at the ICC as counterproductive to the cause of peace," it said. But the ICC begs to differ with the definition of a state for its purposes and refers to the territories as "Palestine." While a preliminary examination is not a formal investigation, it allows the court to review evidence and determine whether to investigate suspects on both sides. Prosecutor Fatou Bensouda said her office would "conduct its analysis in full independence and impartiality." The war between Israel and Hamas militants in Gaza last summer left more than 2,000 people dead. The inquiry will include alleged war crimes committed since June. The International Criminal Court was set up in 2002 to prosecute genocide, crimes against humanity and war crimes. CNN\'s Vasco Cotovio, Kareem Khadder and Faith Karimi contributed to this report.' + + # The below article tests that we don't add any hypotheses outside of the top n_beams + IRAN_ARTICLE = " (CNN)The United States and its negotiating partners reached a very strong framework agreement with Iran in Lausanne, Switzerland, on Thursday that limits Iran's nuclear program in such a way as to effectively block it from building a nuclear weapon. Expect pushback anyway, if the recent past is any harbinger. Just last month, in an attempt to head off such an agreement, House Speaker John Boehner invited Israeli Prime Minister Benjamin Netanyahu to preemptively blast it before Congress, and 47 senators sent a letter to the Iranian leadership warning them away from a deal. The debate that has already begun since the announcement of the new framework will likely result in more heat than light. It will not be helped by the gathering swirl of dubious assumptions and doubtful assertions. Let us address some of these: . The most misleading assertion, despite universal rejection by experts, is that the negotiations' objective at the outset was the total elimination of any nuclear program in Iran. That is the position of Netanyahu and his acolytes in the U.S. Congress. But that is not and never was the objective. If it had been, there would have been no Iranian team at the negotiating table. Rather, the objective has always been to structure an agreement or series of agreements so that Iran could not covertly develop a nuclear arsenal before the United States and its allies could respond. The new framework has exceeded expectations in achieving that goal. It would reduce Iran's low-enriched uranium stockpile, cut by two-thirds its number of installed centrifuges and implement a rigorous inspection regime. Another dubious assumption of opponents is that the Iranian nuclear program is a covert weapons program. Despite sharp accusations by some in the United States and its allies, Iran denies having such a program, and U.S. intelligence contends that Iran has not yet made the decision to build a nuclear weapon. Iran's continued cooperation with International Atomic Energy Agency inspections is further evidence on this point, and we'll know even more about Iran's program in the coming months and years because of the deal. In fact, the inspections provisions that are part of this agreement are designed to protect against any covert action by the Iranians. What's more, the rhetoric of some members of Congress has implied that the negotiations have been between only the United States and Iran (i.e., the 47 senators' letter warning that a deal might be killed by Congress or a future president). This of course is not the case. The talks were between Iran and the five permanent members of the U.N. Security Council (United States, United Kingdom, France, China and Russia) plus Germany, dubbed the P5+1. While the United States has played a leading role in the effort, it negotiated the terms alongside its partners. If the agreement reached by the P5+1 is rejected by Congress, it could result in an unraveling of the sanctions on Iran and threaten NATO cohesion in other areas. Another questionable assertion is that this agreement contains a sunset clause, after which Iran will be free to do as it pleases. Again, this is not the case. Some of the restrictions on Iran's nuclear activities, such as uranium enrichment, will be eased or eliminated over time, as long as 15 years. But most importantly, the framework agreement includes Iran's ratification of the Additional Protocol, which allows IAEA inspectors expanded access to nuclear sites both declared and nondeclared. This provision will be permanent. It does not sunset. Thus, going forward, if Iran decides to enrich uranium to weapons-grade levels, monitors will be able to detect such a move in a matter of days and alert the U.N. Security Council. Many in Congress have said that the agreement should be a formal treaty requiring the Senate to \"advise and consent.\" But the issue is not suited for a treaty. Treaties impose equivalent obligations on all signatories. For example, the New START treaty limits Russia and the United States to 1,550 deployed strategic warheads. But any agreement with Iran will not be so balanced. The restrictions and obligations in the final framework agreement will be imposed almost exclusively on Iran. The P5+1 are obligated only to ease and eventually remove most but not all economic sanctions, which were imposed as leverage to gain this final deal. Finally some insist that any agreement must address Iranian missile programs, human rights violations or support for Hamas or Hezbollah. As important as these issues are, and they must indeed be addressed, they are unrelated to the most important aim of a nuclear deal: preventing a nuclear Iran. To include them in the negotiations would be a poison pill. This agreement should be judged on its merits and on how it affects the security of our negotiating partners and allies, including Israel. Those judgments should be fact-based, not based on questionable assertions or dubious assumptions." + + ARTICLE_SUBWAY = ' New York (CNN)When Liana Barrientos was 23 years old, she got married in Westchester County, New York. A year later, she got married again in Westchester County, but to a different man and without divorcing her first husband. Only 18 days after that marriage, she got hitched yet again. Then, Barrientos declared "I do" five more times, sometimes only within two weeks of each other. In 2010, she married once more, this time in the Bronx. In an application for a marriage license, she stated it was her "first and only" marriage. Barrientos, now 39, is facing two criminal counts of "offering a false instrument for filing in the first degree," referring to her false statements on the 2010 marriage license application, according to court documents. Prosecutors said the marriages were part of an immigration scam. On Friday, she pleaded not guilty at State Supreme Court in the Bronx, according to her attorney, Christopher Wright, who declined to comment further. After leaving court, Barrientos was arrested and charged with theft of service and criminal trespass for allegedly sneaking into the New York subway through an emergency exit, said Detective Annette Markowski, a police spokeswoman. In total, Barrientos has been married 10 times, with nine of her marriages occurring between 1999 and 2002. All occurred either in Westchester County, Long Island, New Jersey or the Bronx. She is believed to still be married to four men, and at one time, she was married to eight men at once, prosecutors say. Prosecutors said the immigration scam involved some of her husbands, who filed for permanent residence status shortly after the marriages. Any divorces happened only after such filings were approved. It was unclear whether any of the men will be prosecuted. The case was referred to the Bronx District Attorney\'s Office by Immigration and Customs Enforcement and the Department of Homeland Security\'s Investigation Division. Seven of the men are from so-called "red-flagged" countries, including Egypt, Turkey, Georgia, Pakistan and Mali. Her eighth husband, Rashid Rajput, was deported in 2006 to his native Pakistan after an investigation by the Joint Terrorism Task Force. If convicted, Barrientos faces up to four years in prison. Her next court appearance is scheduled for May 18.' + + dct = tok.batch_encode_plus( + [FRANCE_ARTICLE, SHORTER_ARTICLE, IRAN_ARTICLE, ARTICLE_SUBWAY], + max_length=1024, + padding="max_length", + truncation_strategy="only_first", + truncation=True, + return_tensors="pt", + ) + + self.assertEqual(1024, dct["input_ids"].shape[1]) + hypotheses_batch = hf.generate( + input_ids=dct["input_ids"].to(torch_device), + attention_mask=dct["attention_mask"].to(torch_device), + num_beams=2, + ) + assert hypotheses_batch[:, 1].eq(0).all().item() + + EXPECTED = [ + "A French prosecutor says he is not aware of any video footage from on board the plane. Two German " + "magazines claim to have found a cell phone video showing the crash. The publications say they watched " + "the video, which was found by a source close to the investigation. All 150 on board Germanwings Flight " + "9525 were killed.", + "Palestinian Authority becomes 123rd member of the International Criminal Court. The move gives the court " + "jurisdiction over alleged crimes in Palestinian territories. Israel and the United States opposed the " + "Palestinians' efforts to join the body. But Palestinian Foreign Minister Riad al-Malki said it was a " + "move toward greater justice.", + "U.S. and its negotiating partners reached a strong framework agreement with Iran. Peter Bergen: The " + "debate that has already begun will likely result in more heat than light. He says critics have made " + "dubious assumptions and doubtful assertions. Bergen says the goal was to block Iran from building a " + "nuclear weapon.", + "Liana Barrientos, 39, has been married 10 times, sometimes within two weeks of each other. Prosecutors " + "say the marriages were part of an immigration scam. She pleaded not guilty at State Supreme Court in the " + "Bronx on Friday. If convicted, she faces up to four years in prison.", + ] + + generated_summaries = tok.batch_decode( + hypotheses_batch.tolist(), clean_up_tokenization_spaces=True, skip_special_tokens=True + ) + assert generated_summaries == EXPECTED + + +class OPTStandaloneDecoderModelTester: + def __init__( + self, + parent, + vocab_size=99, + batch_size=13, + d_model=16, + decoder_seq_length=7, + is_training=True, + is_decoder=True, + use_attention_mask=True, + use_cache=False, + use_labels=True, + decoder_start_token_id=2, + decoder_ffn_dim=32, + decoder_layers=4, + encoder_attention_heads=4, + decoder_attention_heads=4, + max_position_embeddings=30, + is_encoder_decoder=False, + pad_token_id=0, + bos_token_id=1, + eos_token_id=2, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.decoder_seq_length = decoder_seq_length + # For common tests + self.seq_length = self.decoder_seq_length + self.is_training = is_training + self.use_attention_mask = use_attention_mask + self.use_labels = use_labels + + self.vocab_size = vocab_size + self.d_model = d_model + self.hidden_size = d_model + self.num_hidden_layers = decoder_layers + self.decoder_layers = decoder_layers + self.decoder_ffn_dim = decoder_ffn_dim + self.encoder_attention_heads = encoder_attention_heads + self.decoder_attention_heads = decoder_attention_heads + self.num_attention_heads = decoder_attention_heads + self.eos_token_id = eos_token_id + self.bos_token_id = bos_token_id + self.pad_token_id = pad_token_id + self.decoder_start_token_id = decoder_start_token_id + self.use_cache = use_cache + self.max_position_embeddings = max_position_embeddings + self.is_encoder_decoder = is_encoder_decoder + + self.scope = None + self.decoder_key_length = decoder_seq_length + self.base_model_out_len = 2 + self.decoder_attention_idx = 1 + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.decoder_seq_length], self.vocab_size) + + attention_mask = None + if self.use_attention_mask: + attention_mask = ids_tensor([self.batch_size, self.decoder_seq_length], vocab_size=2) + + lm_labels = None + if self.use_labels: + lm_labels = ids_tensor([self.batch_size, self.decoder_seq_length], self.vocab_size) + + config = OPTConfig( + vocab_size=self.vocab_size, + d_model=self.d_model, + encoder_layers=self.decoder_layers, + decoder_layers=self.decoder_layers, + decoder_ffn_dim=self.decoder_ffn_dim, + encoder_attention_heads=self.encoder_attention_heads, + decoder_attention_heads=self.decoder_attention_heads, + eos_token_id=self.eos_token_id, + bos_token_id=self.bos_token_id, + use_cache=self.use_cache, + pad_token_id=self.pad_token_id, + decoder_start_token_id=self.decoder_start_token_id, + max_position_embeddings=self.max_position_embeddings, + is_encoder_decoder=self.is_encoder_decoder, + ) + + return ( + config, + input_ids, + attention_mask, + lm_labels, + ) + + def prepare_config_and_inputs_for_decoder(self): + ( + config, + input_ids, + attention_mask, + lm_labels, + ) = self.prepare_config_and_inputs() + + encoder_hidden_states = floats_tensor([self.batch_size, self.decoder_seq_length, self.hidden_size]) + encoder_attention_mask = ids_tensor([self.batch_size, self.decoder_seq_length], vocab_size=2) + + return ( + config, + input_ids, + attention_mask, + encoder_hidden_states, + encoder_attention_mask, + lm_labels, + ) + + def create_and_check_decoder_model_past( + self, + config, + input_ids, + attention_mask, + lm_labels, + ): + config.use_cache = True + model = OPTDecoder(config=config).to(torch_device).eval() + # first forward pass + outputs = model(input_ids, use_cache=True) + outputs_use_cache_conf = model(input_ids) + outputs_no_past = model(input_ids, use_cache=False) + + self.parent.assertTrue(len(outputs) == len(outputs_use_cache_conf)) + self.parent.assertTrue(len(outputs) == len(outputs_no_past) + 1) + + past_key_values = outputs["past_key_values"] + + # create hypothetical next token and extent to next_input_ids + next_tokens = ids_tensor((self.batch_size, 1), config.vocab_size) + + # append to next input_ids and + next_input_ids = torch.cat([input_ids, next_tokens], dim=-1) + + output_from_no_past = model(next_input_ids)["last_hidden_state"] + output_from_past = model(next_tokens, past_key_values=past_key_values)["last_hidden_state"] + + # select random slice + random_slice_idx = ids_tensor((1,), output_from_past.shape[-1]).item() + output_from_no_past_slice = output_from_no_past[:, next_input_ids.shape[-1] - 1, random_slice_idx].detach() + output_from_past_slice = output_from_past[:, 0, random_slice_idx].detach() + + # test that outputs are equal for slice + assert torch.allclose(output_from_past_slice, output_from_no_past_slice, atol=1e-3) + + def create_and_check_decoder_model_attention_mask_past( + self, + config, + input_ids, + attention_mask, + lm_labels, + ): + model = OPTDecoder(config=config).to(torch_device).eval() + + # create attention mask + attn_mask = torch.ones(input_ids.shape, dtype=torch.long, device=torch_device) + + half_seq_length = input_ids.shape[-1] // 2 + attn_mask[:, half_seq_length:] = 0 + + # first forward pass + past_key_values = model(input_ids, attention_mask=attn_mask, use_cache=True)["past_key_values"] + + # create hypothetical next token and extent to next_input_ids + next_tokens = ids_tensor((self.batch_size, 1), config.vocab_size) + + # change a random masked slice from input_ids + random_seq_idx_to_change = ids_tensor((1,), half_seq_length).item() + 1 + random_other_next_tokens = ids_tensor((self.batch_size, 1), config.vocab_size).squeeze(-1) + input_ids[:, -random_seq_idx_to_change] = random_other_next_tokens + + # append to next input_ids and attn_mask + next_input_ids = torch.cat([input_ids, next_tokens], dim=-1) + attn_mask = torch.cat( + [attn_mask, torch.ones((attn_mask.shape[0], 1), dtype=torch.long, device=torch_device)], + dim=1, + ) + + # get two different outputs + output_from_no_past = model(next_input_ids, attention_mask=attn_mask)["last_hidden_state"] + output_from_past = model(next_tokens, attention_mask=attn_mask, past_key_values=past_key_values)[ + "last_hidden_state" + ] + + # select random slice + random_slice_idx = ids_tensor((1,), output_from_past.shape[-1]).item() + output_from_no_past_slice = output_from_no_past[:, next_input_ids.shape[-1] - 1, random_slice_idx].detach() + output_from_past_slice = output_from_past[:, 0, random_slice_idx].detach() + + # test that outputs are equal for slice + assert torch.allclose(output_from_past_slice, output_from_no_past_slice, atol=1e-3) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + ( + config, + input_ids, + attention_mask, + lm_labels, + ) = config_and_inputs + + inputs_dict = { + "input_ids": input_ids, + "attention_mask": attention_mask, + } + return config, inputs_dict + + +@require_torch +class OPTStandaloneDecoderModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase): + all_model_classes = (OPTDecoder, OPTForCausalLM) if is_torch_available() else () + all_generative_model_classes = (OPTForCausalLM,) if is_torch_available() else () + test_pruning = False + is_encoder_decoder = False + + def setUp( + self, + ): + self.model_tester = OPTStandaloneDecoderModelTester(self, is_training=False) + self.config_tester = ConfigTester(self, config_class=OPTConfig) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_decoder_model_past(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_decoder_model_past(*config_and_inputs) + + def test_decoder_model_attn_mask_past(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_decoder_model_attention_mask_past(*config_and_inputs) + + def test_retain_grad_hidden_states_attentions(self): + # decoder cannot keep gradients + return diff --git a/tests/models/opt/test_modeling_tf_opt.py b/tests/models/opt/test_modeling_tf_opt.py new file mode 100644 index 0000000000000..9d3bb3f7107cd --- /dev/null +++ b/tests/models/opt/test_modeling_tf_opt.py @@ -0,0 +1,477 @@ +# coding=utf-8 +# Copyright 2021 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np + +from transformers import OPTConfig, OPTTokenizer, is_tf_available +from transformers.testing_utils import require_tf, slow +from transformers.utils import cached_property + +from ...test_configuration_common import ConfigTester +from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor +from ...utils.test_modeling_tf_core import TFCoreModelTesterMixin + + +if is_tf_available(): + import tensorflow as tf + + from transformers import TFOPTForConditionalGeneration, TFOPTModel + + +@require_tf +class TFOPTModelTester: + config_cls = OPTConfig + config_updates = {} + hidden_act = "gelu" + + def __init__( + self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_labels=False, + vocab_size=99, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=20, + eos_token_id=2, + pad_token_id=1, + bos_token_id=0, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_labels = use_labels + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.eos_token_id = eos_token_id + self.pad_token_id = pad_token_id + self.bos_token_id = bos_token_id + + def prepare_config_and_inputs_for_common(self): + input_ids = ids_tensor([self.batch_size, self.seq_length - 1], self.vocab_size) + eos_tensor = tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size), 1) + input_ids = tf.concat([input_ids, eos_tensor], axis=1) + + decoder_input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + config = self.config_cls( + vocab_size=self.vocab_size, + d_model=self.hidden_size, + encoder_layers=self.num_hidden_layers, + decoder_layers=self.num_hidden_layers, + encoder_attention_heads=self.num_attention_heads, + decoder_attention_heads=self.num_attention_heads, + encoder_ffn_dim=self.intermediate_size, + decoder_ffn_dim=self.intermediate_size, + dropout=self.hidden_dropout_prob, + attention_dropout=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + eos_token_ids=[2], + bos_token_id=self.bos_token_id, + pad_token_id=self.pad_token_id, + decoder_start_token_id=self.pad_token_id, + **self.config_updates, + ) + inputs_dict = prepare_opt_inputs_dict(config, input_ids, decoder_input_ids) + return config, inputs_dict + + def check_decoder_model_past_large_inputs(self, config, inputs_dict): + model = TFOPTModel(config=config).get_decoder() + input_ids = inputs_dict["input_ids"] + + input_ids = input_ids[:1, :] + attention_mask = inputs_dict["attention_mask"][:1, :] + head_mask = inputs_dict["head_mask"] + self.batch_size = 1 + + # first forward pass + outputs = model(input_ids, attention_mask=attention_mask, head_mask=head_mask, use_cache=True) + + output, past_key_values = outputs.to_tuple() + + # create hypothetical next token and extent to next_input_ids + next_tokens = ids_tensor((self.batch_size, 3), config.vocab_size) + next_attn_mask = tf.cast(ids_tensor((self.batch_size, 3), 2), tf.int8) + + # append to next input_ids and + next_input_ids = tf.concat([input_ids, next_tokens], axis=-1) + next_attention_mask = tf.concat([attention_mask, next_attn_mask], axis=-1) + + output_from_no_past = model(next_input_ids, attention_mask=next_attention_mask)[0] + output_from_past = model(next_tokens, attention_mask=next_attention_mask, past_key_values=past_key_values)[0] + + self.parent.assertEqual(next_tokens.shape[1], output_from_past.shape[1]) + + # select random slice + random_slice_idx = int(ids_tensor((1,), output_from_past.shape[-1])) + output_from_no_past_slice = output_from_no_past[:, -3:, random_slice_idx] + output_from_past_slice = output_from_past[:, :, random_slice_idx] + + # test that outputs are equal for slice + tf.debugging.assert_near(output_from_past_slice, output_from_no_past_slice, rtol=1e-3) + + +def prepare_opt_inputs_dict( + config, + input_ids, + decoder_input_ids, + attention_mask=None, + decoder_attention_mask=None, + head_mask=None, + decoder_head_mask=None, + cross_attn_head_mask=None, +): + if attention_mask is None: + attention_mask = tf.cast(tf.math.not_equal(input_ids, config.pad_token_id), tf.int8) + if decoder_attention_mask is None: + decoder_attention_mask = tf.concat( + [ + tf.ones(decoder_input_ids[:, :1].shape, dtype=tf.int8), + tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:], config.pad_token_id), tf.int8), + ], + axis=-1, + ) + if head_mask is None: + head_mask = tf.ones((config.encoder_layers, config.encoder_attention_heads)) + if decoder_head_mask is None: + decoder_head_mask = tf.ones((config.decoder_layers, config.decoder_attention_heads)) + if cross_attn_head_mask is None: + cross_attn_head_mask = tf.ones((config.decoder_layers, config.decoder_attention_heads)) + return { + "input_ids": input_ids, + "decoder_input_ids": decoder_input_ids, + "attention_mask": attention_mask, + "decoder_attention_mask": decoder_attention_mask, + "head_mask": head_mask, + "decoder_head_mask": decoder_head_mask, + "cross_attn_head_mask": cross_attn_head_mask, + } + + +@require_tf +class TFOPTModelTest(TFModelTesterMixin, TFCoreModelTesterMixin, unittest.TestCase): + all_model_classes = (TFOPTForConditionalGeneration, TFOPTModel) if is_tf_available() else () + all_generative_model_classes = (TFOPTForConditionalGeneration,) if is_tf_available() else () + is_encoder_decoder = True + test_pruning = False + test_onnx = True + onnx_min_opset = 10 + + def setUp(self): + self.model_tester = TFOPTModelTester(self) + self.config_tester = ConfigTester(self, config_class=OPTConfig) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_decoder_model_past_large_inputs(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs_for_common() + self.model_tester.check_decoder_model_past_large_inputs(*config_and_inputs) + + def test_model_common_attributes(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + assert isinstance(model.get_input_embeddings(), tf.keras.layers.Layer) + + if model_class in self.all_generative_model_classes: + x = model.get_output_embeddings() + assert isinstance(x, tf.keras.layers.Layer) + name = model.get_bias() + assert isinstance(name, dict) + for k, v in name.items(): + assert isinstance(v, tf.Variable) + else: + x = model.get_output_embeddings() + assert x is None + name = model.get_bias() + assert name is None + + def test_resize_token_embeddings(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + def _get_word_embedding_weight(model, embedding_layer): + if hasattr(embedding_layer, "weight"): + return embedding_layer.weight + else: + # Here we build the word embeddings weights if not exists. + # And then we retry to get the attribute once built. + model(model.dummy_inputs) + if hasattr(embedding_layer, "weight"): + return embedding_layer.weight + else: + return None + + for model_class in self.all_model_classes: + for size in [config.vocab_size - 10, config.vocab_size + 10, None]: + # build the embeddings + model = model_class(config=config) + old_input_embeddings = _get_word_embedding_weight(model, model.get_input_embeddings()) + old_output_embeddings = _get_word_embedding_weight(model, model.get_output_embeddings()) + old_final_logits_bias = model.get_bias() + + # reshape the embeddings + model.resize_token_embeddings(size) + new_input_embeddings = _get_word_embedding_weight(model, model.get_input_embeddings()) + new_output_embeddings = _get_word_embedding_weight(model, model.get_output_embeddings()) + new_final_logits_bias = model.get_bias() + + # check that the resized embeddings size matches the desired size. + assert_size = size if size is not None else config.vocab_size + + self.assertEqual(new_input_embeddings.shape[0], assert_size) + + # check that weights remain the same after resizing + models_equal = True + for p1, p2 in zip(old_input_embeddings.value(), new_input_embeddings.value()): + if tf.math.reduce_sum(tf.math.abs(p1 - p2)) > 0: + models_equal = False + self.assertTrue(models_equal) + + if old_output_embeddings is not None and new_output_embeddings is not None: + self.assertEqual(new_output_embeddings.shape[0], assert_size) + + models_equal = True + for p1, p2 in zip(old_output_embeddings.value(), new_output_embeddings.value()): + if tf.math.reduce_sum(tf.math.abs(p1 - p2)) > 0: + models_equal = False + self.assertTrue(models_equal) + + if old_final_logits_bias is not None and new_final_logits_bias is not None: + old_final_logits_bias = old_final_logits_bias["final_logits_bias"] + new_final_logits_bias = new_final_logits_bias["final_logits_bias"] + self.assertEqual(new_final_logits_bias.shape[0], 1) + self.assertEqual(new_final_logits_bias.shape[1], assert_size) + + models_equal = True + for old, new in zip(old_final_logits_bias.value(), new_final_logits_bias.value()): + for p1, p2 in zip(old, new): + if tf.math.reduce_sum(tf.math.abs(p1 - p2)) > 0: + models_equal = False + self.assertTrue(models_equal) + + def test_saved_model_creation(self): + # This test is too long (>30sec) and makes fail the CI + pass + + +def _assert_tensors_equal(a, b, atol=1e-12, prefix=""): + """If tensors not close, or a and b arent both tensors, raise a nice Assertion error.""" + if a is None and b is None: + return True + try: + if tf.debugging.assert_near(a, b, atol=atol): + return True + raise + except Exception: + if len(prefix) > 0: + prefix = f"{prefix}: " + raise AssertionError(f"{prefix}{a} != {b}") + + +def _long_tensor(tok_lst): + return tf.constant(tok_lst, dtype=tf.int32) + + +@require_tf +class TFOPTHeadTests(unittest.TestCase): + vocab_size = 99 + + def _get_config_and_data(self): + eos_column_vector = tf.ones((4, 1), dtype=tf.int32) * 2 + input_ids = tf.concat([ids_tensor((4, 6), self.vocab_size - 3) + 3, eos_column_vector], axis=1) + batch_size = input_ids.shape[0] + config = OPTConfig( + vocab_size=self.vocab_size, + d_model=24, + encoder_layers=2, + decoder_layers=2, + encoder_attention_heads=2, + decoder_attention_heads=2, + encoder_ffn_dim=32, + decoder_ffn_dim=32, + max_position_embeddings=48, + eos_token_id=2, + pad_token_id=1, + bos_token_id=0, + decoder_start_token_id=2, + ) + return config, input_ids, batch_size + + def test_lm_forward(self): + config, input_ids, batch_size = self._get_config_and_data() + decoder_lm_labels = ids_tensor([batch_size, input_ids.shape[1]], self.vocab_size) + lm_model = TFOPTForConditionalGeneration(config) + outputs = lm_model(input_ids=input_ids, labels=decoder_lm_labels, decoder_input_ids=input_ids, use_cache=False) + expected_shape = (batch_size, input_ids.shape[1], config.vocab_size) + self.assertEqual(outputs.logits.shape, expected_shape) + + def test_lm_uneven_forward(self): + config = OPTConfig( + vocab_size=10, + d_model=24, + encoder_layers=2, + decoder_layers=2, + encoder_attention_heads=2, + decoder_attention_heads=2, + encoder_ffn_dim=32, + decoder_ffn_dim=32, + max_position_embeddings=48, + ) + lm_model = TFOPTForConditionalGeneration(config) + context = tf.fill((7, 2), 4) + summary = tf.fill((7, 7), 6) + outputs = lm_model(input_ids=context, decoder_input_ids=summary, use_cache=False) + expected_shape = (*summary.shape, config.vocab_size) + self.assertEqual(outputs.logits.shape, expected_shape) + + +@slow +@require_tf +class TFOPTModelIntegrationTest(unittest.TestCase): + def test_inference_no_head(self): + model = TFOPTForConditionalGeneration.from_pretrained("facebook/opt-large").model + + input_ids = _long_tensor([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]) + attention_mask = tf.cast(tf.math.not_equal(input_ids, model.config.pad_token_id), tf.int8) + output = model(input_ids=input_ids, attention_mask=attention_mask)[0] + expected_shape = (1, 11, 1024) + self.assertEqual(output.shape, expected_shape) + expected_slice = tf.convert_to_tensor( + [[0.7144, 0.8143, -1.2813], [0.7144, 0.8143, -1.2813], [-0.0467, 2.5911, -2.1845]], + ) + tf.debugging.assert_near(output[:, :3, :3], expected_slice, atol=1e-3) + + def test_cnn_summarization_same_as_fairseq_hard(self): + hf = TFOPTForConditionalGeneration.from_pretrained("facebook/opt-large-cnn") + tok = self.tok + + FRANCE_ARTICLE = ' Marseille, France (CNN)The French prosecutor leading an investigation into the crash of Germanwings Flight 9525 insisted Wednesday that he was not aware of any video footage from on board the plane. Marseille prosecutor Brice Robin told CNN that "so far no videos were used in the crash investigation." He added, "A person who has such a video needs to immediately give it to the investigators." Robin\'s comments follow claims by two magazines, German daily Bild and French Paris Match, of a cell phone video showing the harrowing final seconds from on board Germanwings Flight 9525 as it crashed into the French Alps. All 150 on board were killed. Paris Match and Bild reported that the video was recovered from a phone at the wreckage site. The two publications described the supposed video, but did not post it on their websites. The publications said that they watched the video, which was found by a source close to the investigation. "One can hear cries of \'My God\' in several languages," Paris Match reported. "Metallic banging can also be heard more than three times, perhaps of the pilot trying to open the cockpit door with a heavy object. Towards the end, after a heavy shake, stronger than the others, the screaming intensifies. Then nothing." "It is a very disturbing scene," said Julian Reichelt, editor-in-chief of Bild online. An official with France\'s accident investigation agency, the BEA, said the agency is not aware of any such video. Lt. Col. Jean-Marc Menichini, a French Gendarmerie spokesman in charge of communications on rescue efforts around the Germanwings crash site, told CNN that the reports were "completely wrong" and "unwarranted." Cell phones have been collected at the site, he said, but that they "hadn\'t been exploited yet." Menichini said he believed the cell phones would need to be sent to the Criminal Research Institute in Rosny sous-Bois, near Paris, in order to be analyzed by specialized technicians working hand-in-hand with investigators. But none of the cell phones found so far have been sent to the institute, Menichini said. Asked whether staff involved in the search could have leaked a memory card to the media, Menichini answered with a categorical "no." Reichelt told "Erin Burnett: Outfront" that he had watched the video and stood by the report, saying Bild and Paris Match are "very confident" that the clip is real. He noted that investigators only revealed they\'d recovered cell phones from the crash site after Bild and Paris Match published their reports. "That is something we did not know before. ... Overall we can say many things of the investigation weren\'t revealed by the investigation at the beginning," he said. What was mental state of Germanwings co-pilot? German airline Lufthansa confirmed Tuesday that co-pilot Andreas Lubitz had battled depression years before he took the controls of Germanwings Flight 9525, which he\'s accused of deliberately crashing last week in the French Alps. Lubitz told his Lufthansa flight training school in 2009 that he had a "previous episode of severe depression," the airline said Tuesday. Email correspondence between Lubitz and the school discovered in an internal investigation, Lufthansa said, included medical documents he submitted in connection with resuming his flight training. The announcement indicates that Lufthansa, the parent company of Germanwings, knew of Lubitz\'s battle with depression, allowed him to continue training and ultimately put him in the cockpit. Lufthansa, whose CEO Carsten Spohr previously said Lubitz was 100% fit to fly, described its statement Tuesday as a "swift and seamless clarification" and said it was sharing the information and documents -- including training and medical records -- with public prosecutors. Spohr traveled to the crash site Wednesday, where recovery teams have been working for the past week to recover human remains and plane debris scattered across a steep mountainside. He saw the crisis center set up in Seyne-les-Alpes, laid a wreath in the village of Le Vernet, closer to the crash site, where grieving families have left flowers at a simple stone memorial. Menichini told CNN late Tuesday that no visible human remains were left at the site but recovery teams would keep searching. French President Francois Hollande, speaking Tuesday, said that it should be possible to identify all the victims using DNA analysis by the end of the week, sooner than authorities had previously suggested. In the meantime, the recovery of the victims\' personal belongings will start Wednesday, Menichini said. Among those personal belongings could be more cell phones belonging to the 144 passengers and six crew on board. Check out the latest from our correspondents . The details about Lubitz\'s correspondence with the flight school during his training were among several developments as investigators continued to delve into what caused the crash and Lubitz\'s possible motive for downing the jet. A Lufthansa spokesperson told CNN on Tuesday that Lubitz had a valid medical certificate, had passed all his examinations and "held all the licenses required." Earlier, a spokesman for the prosecutor\'s office in Dusseldorf, Christoph Kumpa, said medical records reveal Lubitz suffered from suicidal tendencies at some point before his aviation career and underwent psychotherapy before he got his pilot\'s license. Kumpa emphasized there\'s no evidence suggesting Lubitz was suicidal or acting aggressively before the crash. Investigators are looking into whether Lubitz feared his medical condition would cause him to lose his pilot\'s license, a European government official briefed on the investigation told CNN on Tuesday. While flying was "a big part of his life," the source said, it\'s only one theory being considered. Another source, a law enforcement official briefed on the investigation, also told CNN that authorities believe the primary motive for Lubitz to bring down the plane was that he feared he would not be allowed to fly because of his medical problems. Lubitz\'s girlfriend told investigators he had seen an eye doctor and a neuropsychologist, both of whom deemed him unfit to work recently and concluded he had psychological issues, the European government official said. But no matter what details emerge about his previous mental health struggles, there\'s more to the story, said Brian Russell, a forensic psychologist. "Psychology can explain why somebody would turn rage inward on themselves about the fact that maybe they weren\'t going to keep doing their job and they\'re upset about that and so they\'re suicidal," he said. "But there is no mental illness that explains why somebody then feels entitled to also take that rage and turn it outward on 149 other people who had nothing to do with the person\'s problems." Germanwings crash compensation: What we know . Who was the captain of Germanwings Flight 9525? CNN\'s Margot Haddad reported from Marseille and Pamela Brown from Dusseldorf, while Laura Smith-Spark wrote from London. CNN\'s Frederik Pleitgen, Pamela Boykoff, Antonia Mortensen, Sandrine Amiel and Anna-Maja Rappard contributed to this report.' # @noqa + EXPECTED_SUMMARY_FRANCE = 'French prosecutor says he\'s not aware of any video footage from on board the plane. German daily Bild and French Paris Match claim to have found a cell phone video of the crash. A French Gendarmerie spokesman calls the reports "completely wrong" and "unwarranted" German airline Lufthansa confirms co-pilot Andreas Lubitz had battled depression.' + + SHORTER_ARTICLE = ' (CNN)The Palestinian Authority officially became the 123rd member of the International Criminal Court on Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian territories. The formal accession was marked with a ceremony at The Hague, in the Netherlands, where the court is based. The Palestinians signed the ICC\'s founding Rome Statute in January, when they also accepted its jurisdiction over alleged crimes committed "in the occupied Palestinian territory, including East Jerusalem, since June 13, 2014." Later that month, the ICC opened a preliminary examination into the situation in Palestinian territories, paving the way for possible war crimes investigations against Israelis. As members of the court, Palestinians may be subject to counter-charges as well. Israel and the United States, neither of which is an ICC member, opposed the Palestinians\' efforts to join the body. But Palestinian Foreign Minister Riad al-Malki, speaking at Wednesday\'s ceremony, said it was a move toward greater justice. "As Palestine formally becomes a State Party to the Rome Statute today, the world is also a step closer to ending a long era of impunity and injustice," he said, according to an ICC news release. "Indeed, today brings us closer to our shared goals of justice and peace." Judge Kuniko Ozaki, a vice president of the ICC, said acceding to the treaty was just the first step for the Palestinians. "As the Rome Statute today enters into force for the State of Palestine, Palestine acquires all the rights as well as responsibilities that come with being a State Party to the Statute. These are substantive commitments, which cannot be taken lightly," she said. Rights group Human Rights Watch welcomed the development. "Governments seeking to penalize Palestine for joining the ICC should immediately end their pressure, and countries that support universal acceptance of the court\'s treaty should speak out to welcome its membership," said Balkees Jarrah, international justice counsel for the group. "What\'s objectionable is the attempts to undermine international justice, not Palestine\'s decision to join a treaty to which over 100 countries around the world are members." In January, when the preliminary ICC examination was opened, Israeli Prime Minister Benjamin Netanyahu described it as an outrage, saying the court was overstepping its boundaries. The United States also said it "strongly" disagreed with the court\'s decision. "As we have said repeatedly, we do not believe that Palestine is a state and therefore we do not believe that it is eligible to join the ICC," the State Department said in a statement. It urged the warring sides to resolve their differences through direct negotiations. "We will continue to oppose actions against Israel at the ICC as counterproductive to the cause of peace," it said. But the ICC begs to differ with the definition of a state for its purposes and refers to the territories as "Palestine." While a preliminary examination is not a formal investigation, it allows the court to review evidence and determine whether to investigate suspects on both sides. Prosecutor Fatou Bensouda said her office would "conduct its analysis in full independence and impartiality." The war between Israel and Hamas militants in Gaza last summer left more than 2,000 people dead. The inquiry will include alleged war crimes committed since June. The International Criminal Court was set up in 2002 to prosecute genocide, crimes against humanity and war crimes. CNN\'s Vasco Cotovio, Kareem Khadder and Faith Karimi contributed to this report.' + EXPECTED_SUMMARY_SHORTER = "The Palestinian Authority becomes the 123rd member of the International Criminal Court. The move gives the court jurisdiction over alleged crimes in Palestinian territories. Israel and the United States opposed the Palestinians' efforts to join the body. But Palestinian Foreign Minister Riad al-Malki said it was a move toward greater justice." + + # The below article tests that we don't add any hypotheses outside of the top n_beams + IRAN_ARTICLE = " (CNN)The United States and its negotiating partners reached a very strong framework agreement with Iran in Lausanne, Switzerland, on Thursday that limits Iran's nuclear program in such a way as to effectively block it from building a nuclear weapon. Expect pushback anyway, if the recent past is any harbinger. Just last month, in an attempt to head off such an agreement, House Speaker John Boehner invited Israeli Prime Minister Benjamin Netanyahu to preemptively blast it before Congress, and 47 senators sent a letter to the Iranian leadership warning them away from a deal. The debate that has already begun since the announcement of the new framework will likely result in more heat than light. It will not be helped by the gathering swirl of dubious assumptions and doubtful assertions. Let us address some of these: . The most misleading assertion, despite universal rejection by experts, is that the negotiations' objective at the outset was the total elimination of any nuclear program in Iran. That is the position of Netanyahu and his acolytes in the U.S. Congress. But that is not and never was the objective. If it had been, there would have been no Iranian team at the negotiating table. Rather, the objective has always been to structure an agreement or series of agreements so that Iran could not covertly develop a nuclear arsenal before the United States and its allies could respond. The new framework has exceeded expectations in achieving that goal. It would reduce Iran's low-enriched uranium stockpile, cut by two-thirds its number of installed centrifuges and implement a rigorous inspection regime. Another dubious assumption of opponents is that the Iranian nuclear program is a covert weapons program. Despite sharp accusations by some in the United States and its allies, Iran denies having such a program, and U.S. intelligence contends that Iran has not yet made the decision to build a nuclear weapon. Iran's continued cooperation with International Atomic Energy Agency inspections is further evidence on this point, and we'll know even more about Iran's program in the coming months and years because of the deal. In fact, the inspections provisions that are part of this agreement are designed to protect against any covert action by the Iranians. What's more, the rhetoric of some members of Congress has implied that the negotiations have been between only the United States and Iran (i.e., the 47 senators' letter warning that a deal might be killed by Congress or a future president). This of course is not the case. The talks were between Iran and the five permanent members of the U.N. Security Council (United States, United Kingdom, France, China and Russia) plus Germany, dubbed the P5+1. While the United States has played a leading role in the effort, it negotiated the terms alongside its partners. If the agreement reached by the P5+1 is rejected by Congress, it could result in an unraveling of the sanctions on Iran and threaten NATO cohesion in other areas. Another questionable assertion is that this agreement contains a sunset clause, after which Iran will be free to do as it pleases. Again, this is not the case. Some of the restrictions on Iran's nuclear activities, such as uranium enrichment, will be eased or eliminated over time, as long as 15 years. But most importantly, the framework agreement includes Iran's ratification of the Additional Protocol, which allows IAEA inspectors expanded access to nuclear sites both declared and nondeclared. This provision will be permanent. It does not sunset. Thus, going forward, if Iran decides to enrich uranium to weapons-grade levels, monitors will be able to detect such a move in a matter of days and alert the U.N. Security Council. Many in Congress have said that the agreement should be a formal treaty requiring the Senate to \"advise and consent.\" But the issue is not suited for a treaty. Treaties impose equivalent obligations on all signatories. For example, the New START treaty limits Russia and the United States to 1,550 deployed strategic warheads. But any agreement with Iran will not be so balanced. The restrictions and obligations in the final framework agreement will be imposed almost exclusively on Iran. The P5+1 are obligated only to ease and eventually remove most but not all economic sanctions, which were imposed as leverage to gain this final deal. Finally some insist that any agreement must address Iranian missile programs, human rights violations or support for Hamas or Hezbollah. As important as these issues are, and they must indeed be addressed, they are unrelated to the most important aim of a nuclear deal: preventing a nuclear Iran. To include them in the negotiations would be a poison pill. This agreement should be judged on its merits and on how it affects the security of our negotiating partners and allies, including Israel. Those judgments should be fact-based, not based on questionable assertions or dubious assumptions." + EXPECTED_SUMMARY_IRAN = "The U.S. and its negotiating partners reached a very strong framework agreement with Iran. Peter Bergen: The debate that has already begun will likely result in more heat than light. He says the agreement limits Iran's nuclear program in such a way as to effectively block it from building a nuclear weapon. Bergen says the most important aim of a nuclear deal is preventing a nuclear Iran." + + ARTICLE_SUBWAY = ' New York (CNN)When Liana Barrientos was 23 years old, she got married in Westchester County, New York. A year later, she got married again in Westchester County, but to a different man and without divorcing her first husband. Only 18 days after that marriage, she got hitched yet again. Then, Barrientos declared "I do" five more times, sometimes only within two weeks of each other. In 2010, she married once more, this time in the Bronx. In an application for a marriage license, she stated it was her "first and only" marriage. Barrientos, now 39, is facing two criminal counts of "offering a false instrument for filing in the first degree," referring to her false statements on the 2010 marriage license application, according to court documents. Prosecutors said the marriages were part of an immigration scam. On Friday, she pleaded not guilty at State Supreme Court in the Bronx, according to her attorney, Christopher Wright, who declined to comment further. After leaving court, Barrientos was arrested and charged with theft of service and criminal trespass for allegedly sneaking into the New York subway through an emergency exit, said Detective Annette Markowski, a police spokeswoman. In total, Barrientos has been married 10 times, with nine of her marriages occurring between 1999 and 2002. All occurred either in Westchester County, Long Island, New Jersey or the Bronx. She is believed to still be married to four men, and at one time, she was married to eight men at once, prosecutors say. Prosecutors said the immigration scam involved some of her husbands, who filed for permanent residence status shortly after the marriages. Any divorces happened only after such filings were approved. It was unclear whether any of the men will be prosecuted. The case was referred to the Bronx District Attorney\'s Office by Immigration and Customs Enforcement and the Department of Homeland Security\'s Investigation Division. Seven of the men are from so-called "red-flagged" countries, including Egypt, Turkey, Georgia, Pakistan and Mali. Her eighth husband, Rashid Rajput, was deported in 2006 to his native Pakistan after an investigation by the Joint Terrorism Task Force. If convicted, Barrientos faces up to four years in prison. Her next court appearance is scheduled for May 18.' + EXPECTED_SUMMARY_SUBWAY = "Liana Barrientos has been married 10 times, sometimes within two weeks of each other. Prosecutors say the marriages were part of an immigration scam. On Friday, she pleaded not guilty at State Supreme Court in the Bronx. She was arrested and charged with theft of service and criminal trespass for allegedly sneaking into the subway." + + dct = tok( + [FRANCE_ARTICLE, SHORTER_ARTICLE, IRAN_ARTICLE, ARTICLE_SUBWAY], + max_length=1024, + truncation_strategy="only_first", + padding="longest", + truncation=True, + return_tensors="tf", + ) + self.assertEqual(1024, dct["input_ids"].shape[1]) + hypotheses_batch = hf.generate( + input_ids=dct["input_ids"], + attention_mask=dct["attention_mask"], + ) + + assert hypotheses_batch[:, 1].numpy().tolist() == [0, 0, 0, 0] # test force_bos_token_to_be_generated + decoded = tok.batch_decode(hypotheses_batch, skip_special_tokens=True, clean_up_tokenization_spaces=False) + expected_batch = [ + EXPECTED_SUMMARY_FRANCE, + EXPECTED_SUMMARY_SHORTER, + EXPECTED_SUMMARY_IRAN, + EXPECTED_SUMMARY_SUBWAY, + ] + assert decoded == expected_batch + + @cached_property + def tok(self): + return OPTTokenizer.from_pretrained("facebook/opt-large") + + +@slow +@require_tf +class FasterTFOPTModelIntegrationTests(unittest.TestCase): + """These tests are useful for debugging since they operate on a model with 1 encoder layer and 1 decoder layer.""" + + @cached_property + def tok(self): + return OPTTokenizer.from_pretrained("facebook/opt-large") + + @cached_property + def xsum_1_1_model(self): + return TFOPTForConditionalGeneration.from_pretrained("sshleifer/distilopt-xsum-1-1") + + def test_xsum_1_1_generation(self): + model = self.xsum_1_1_model + assert model.model.decoder.embed_tokens._layer == model.model.shared + ARTICLE = 'The Palestinian Authority officially became the 123rd member of the International Criminal Court on Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian territories. The formal accession was marked with a ceremony at The Hague, in the Netherlands, where the court is based. The Palestinians signed the ICC\'s founding Rome Statute in January, when they also accepted its jurisdiction over alleged crimes committed "in the occupied Palestinian territory, including East Jerusalem, since June 13, 2014." Later that month, the ICC opened a preliminary examination into the situation in Palestinian territories, paving the way for possible war crimes investigations against Israelis. As members of the court, Palestinians may be subject to counter-charges as well. Israel and the United States, neither of which is an ICC member, opposed the Palestinians\' efforts to join the body. But Palestinian Foreign Minister Riad al-Malki, speaking at Wednesday\'s ceremony, said it was a move toward greater justice. "As Palestine formally becomes a State Party to the Rome Statute today, the world is also a step closer to ending a long era of impunity and injustice," he said, according to an ICC news release. "Indeed, today brings us closer to our shared goals of justice and peace." Judge Kuniko Ozaki, a vice president of the ICC, said acceding to the treaty was just the first step for the Palestinians. "As the Rome Statute today enters into force for the State of Palestine, Palestine acquires all the rights as well as responsibilities that come with being a State Party to the Statute. These are substantive commitments, which cannot be taken lightly," she said. Rights group Human Rights Watch welcomed the development. "Governments seeking to penalize Palestine for joining the ICC should immediately end their pressure, and countries that support universal acceptance of the court\'s treaty should speak out to welcome its membership," said Balkees Jarrah, international justice counsel for the group. "What\'s objectionable is the attempts to undermine international justice, not Palestine\'s decision to join a treaty to which over 100 countries around the world are members." In January, when the preliminary ICC examination was opened, Israeli Prime Minister Benjamin Netanyahu described it as an outrage, saying the court was overstepping its boundaries. The United States also said it "strongly" disagreed with the court\'s decision. "As we have said repeatedly, we do not believe that Palestine is a state and therefore we do not believe that it is eligible to join the ICC," the State Department said in a statement. It urged the warring sides to resolve their differences through direct negotiations. "We will continue to oppose actions against Israel at the ICC as counterproductive to the cause of peace," it said. But the ICC begs to differ with the definition of a state for its purposes and refers to the territories as "Palestine." While a preliminary examination is not a formal investigation, it allows the court to review evidence and determine whether to investigate suspects on both sides. Prosecutor Fatou Bensouda said her office would "conduct its analysis in full independence and impartiality." The war between Israel and Hamas militants in Gaza last summer left more than 2,000 people dead. The inquiry will include alleged war crimes committed since June. The International Criminal Court was set up in 2002 to prosecute genocide, crimes against humanity and war crimes.' + EXPECTED = " The International Criminal Court (ICC) has announced that it has been announced by the International Criminal court." + dct = self.tok(ARTICLE, return_tensors="tf") + generated_ids = model.generate(**dct, num_beams=4) + result = self.tok.batch_decode(generated_ids, skip_special_tokens=True)[0] + assert result == EXPECTED + + def test_xsum_1_1_batch_generation(self): + batch = self.tok( + [ + 'The Palestinian Authority officially became the 123rd member of the International Criminal Court on Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian territories. The formal accession was marked with a ceremony at The Hague, in the Netherlands, where the court is based. The Palestinians signed the ICC\'s founding Rome Statute in January, when they also accepted its jurisdiction over alleged crimes committed "in the occupied Palestinian territory, including East Jerusalem, since June 13, 2014." Later that month, the ICC opened a preliminary examination into the situation in Palestinian territories, paving the way for possible war crimes investigations against Israelis. As members of the court, Palestinians may be subject to counter-charges as well. Israel and the United States, neither of which is an ICC member, opposed the Palestinians\' efforts to join the body. But Palestinian Foreign Minister Riad al-Malki, speaking at Wednesday\'s ceremony, said it was a move toward greater justice. "As Palestine formally becomes a State Party to the Rome Statute today, the world is also a step closer to ending a long era of impunity and injustice," he said, according to an ICC news release. "Indeed, today brings us closer to our shared goals of justice and peace." Judge Kuniko Ozaki, a vice president of the ICC, said acceding to the treaty was just the first step for the Palestinians. "As the Rome Statute today enters into force for the State of Palestine, Palestine acquires all the rights as well as responsibilities that come with being a State Party to the Statute. These are substantive commitments, which cannot be taken lightly," she said. Rights group Human Rights Watch welcomed the development. "Governments seeking to penalize Palestine for joining the ICC should immediately end their pressure, and countries that support universal acceptance of the court\'s treaty should speak out to welcome its membership," said Balkees Jarrah, international justice counsel for the group. "What\'s objectionable is the attempts to undermine international justice, not Palestine\'s decision to join a treaty to which over 100 countries around the world are members." In January, when the preliminary ICC examination was opened, Israeli Prime Minister Benjamin Netanyahu described it as an outrage, saying the court was overstepping its boundaries. The United States also said it "strongly" disagreed with the court\'s decision. "As we have said repeatedly, we do not believe that Palestine is a state and therefore we do not believe that it is eligible to join the ICC," the State Department said in a statement. It urged the warring sides to resolve their differences through direct negotiations. "We will continue to oppose actions against Israel at the ICC as counterproductive to the cause of peace," it said. But the ICC begs to differ with the definition of a state for its purposes and refers to the territories as "Palestine." While a preliminary examination is not a formal investigation, it allows the court to review evidence and determine whether to investigate suspects on both sides. Prosecutor Fatou Bensouda said her office would "conduct its analysis in full independence and impartiality." The war between Israel and Hamas militants in Gaza last summer left more than 2,000 people dead. The inquiry will include alleged war crimes committed since June. The International Criminal Court was set up in 2002 to prosecute genocide, crimes against humanity and war crimes.', + 'The French prosecutor leading an investigation into the crash of Germanwings Flight 9525 insisted Wednesday that he was not aware of any video footage from on board the plane. Marseille prosecutor Brice Robin told CNN that "so far no videos were used in the crash investigation." He added, "A person who has such a video needs to immediately give it to the investigators." Robin\'s comments follow claims by two magazines, German daily Bild and French Paris Match, of a cell phone video showing the harrowing final seconds from on board Germanwings Flight 9525 as it crashed into the French Alps. All 150 on board were killed. Paris Match and Bild reported that the video was recovered from a phone at the wreckage site. The two publications described the supposed video, but did not post it on their websites. The publications said that they watched the video, which was found by a source close to the investigation. "One can hear cries of \'My God\' in several languages," Paris Match reported. "Metallic banging can also be heard more than three times, perhaps of the pilot trying to open the cockpit door with a heavy object. Towards the end, after a heavy shake, stronger than the others, the screaming intensifies. Then nothing." "It is a very disturbing scene," said Julian Reichelt, editor-in-chief of Bild online. An official with France\'s accident investigation agency, the BEA, said the agency is not aware of any such video. Lt. Col. Jean-Marc Menichini, a French Gendarmerie spokesman in charge of communications on rescue efforts around the Germanwings crash site, told CNN that the reports were "completely wrong" and "unwarranted." Cell phones have been collected at the site, he said, but that they "hadn\'t been exploited yet." Menichini said he believed the cell phones would need to be sent to the Criminal Research Institute in Rosny sous-Bois, near Paris, in order to be analyzed by specialized technicians working hand-in-hand with investigators. But none of the cell phones found so far have been sent to the institute, Menichini said. Asked whether staff involved in the search could have leaked a memory card to the media, Menichini answered with a categorical "no." Reichelt told "Erin Burnett: Outfront" that he had watched the video and stood by the report, saying Bild and Paris Match are "very confident" that the clip is real. He noted that investigators only revealed they\'d recovered cell phones from the crash site after Bild and Paris Match published their reports. "That is something we did not know before. ... Overall we can say many things of the investigation weren\'t revealed by the investigation at the beginning," he said. What was mental state of Germanwings co-pilot? German airline Lufthansa confirmed Tuesday that co-pilot Andreas Lubitz had battled depression years before he took the controls of Germanwings Flight 9525, which he\'s accused of deliberately crashing last week in the French Alps. Lubitz told his Lufthansa flight training school in 2009 that he had a "previous episode of severe depression," the airline said Tuesday. Email correspondence between Lubitz and the school discovered in an internal investigation, Lufthansa said, included medical documents he submitted in connection with resuming his flight training. The announcement indicates that Lufthansa, the parent company of Germanwings, knew of Lubitz\'s battle with depression, allowed him to continue training and ultimately put him in the cockpit. Lufthansa, whose CEO Carsten Spohr previously said Lubitz was 100% fit to fly, described its statement Tuesday as a "swift and seamless clarification" and said it was sharing the information and documents -- including training and medical records -- with public prosecutors. Spohr traveled to the crash site Wednesday, where recovery teams have been working for the past week to recover human remains and plane debris scattered across a steep mountainside. He saw the crisis center set up in Seyne-les-Alpes, laid a wreath in the village of Le Vernet, closer to the crash site, where grieving families have left flowers at a simple stone memorial. Menichini told CNN late Tuesday that no visible human remains were left at the site but recovery teams would keep searching. French President Francois Hollande, speaking Tuesday, said that it should be possible to identify all the victims using DNA analysis by the end of the week, sooner than authorities had previously suggested. In the meantime, the recovery of the victims\' personal belongings will start Wednesday, Menichini said. Among those personal belongings could be more cell phones belonging to the 144 passengers and six crew on board. Check out the latest from our correspondents . The details about Lubitz\'s correspondence with the flight school during his training were among several developments as investigators continued to delve into what caused the crash and Lubitz\'s possible motive for downing the jet. A Lufthansa spokesperson told CNN on Tuesday that Lubitz had a valid medical certificate, had passed all his examinations and "held all the licenses required." Earlier, a spokesman for the prosecutor\'s office in Dusseldorf, Christoph Kumpa, said medical records reveal Lubitz suffered from suicidal tendencies at some point before his aviation career and underwent psychotherapy before he got his pilot\'s license. Kumpa emphasized there\'s no evidence suggesting Lubitz was suicidal or acting aggressively before the crash. Investigators are looking into whether Lubitz feared his medical condition would cause him to lose his pilot\'s license, a European government official briefed on the investigation told CNN on Tuesday. While flying was "a big part of his life," the source said, it\'s only one theory being considered. Another source, a law enforcement official briefed on the investigation, also told CNN that authorities believe the primary motive for Lubitz to bring down the plane was that he feared he would not be allowed to fly because of his medical problems. Lubitz\'s girlfriend told investigators he had seen an eye doctor and a neuropsychologist, both of whom deemed him unfit to work recently and concluded he had psychological issues, the European government official said. But no matter what details emerge about his previous mental health struggles, there\'s more to the story, said Brian Russell, a forensic psychologist. "Psychology can explain why somebody would turn rage inward on themselves about the fact that maybe they weren\'t going to keep doing their job and they\'re upset about that and so they\'re suicidal," he said. "But there is no mental illness that explains why somebody then feels entitled to also take that rage and turn it outward on 149 other people who had nothing to do with the person\'s problems." Germanwings crash compensation: What we know . Who was the captain of Germanwings Flight 9525? CNN\'s Margot Haddad reported from Marseille and Pamela Brown from Dusseldorf, while Laura Smith-Spark wrote from London. CNN\'s Frederik Pleitgen, Pamela Boykoff, Antonia Mortensen, Sandrine Amiel and Anna-Maja Rappard contributed to this report.', + ], + return_tensors="tf", + padding="longest", + truncation=True, + ) + generated_ids = self.xsum_1_1_model.generate(**batch, num_beams=4) + result = self.tok.batch_decode(generated_ids, skip_special_tokens=True) + assert ( + result[0] + == " The International Criminal Court (ICC) has announced that it has been announced by the International Criminal court." + ) + assert ( + result[1] + == " An investigation into the crash that killed at least 10 people in the French capital has been released by the French police investigating the crash." + ) + + def test_encoder_equiv(self): + batch = self.tok( + [ + 'The Palestinian Authority officially became the 123rd member of the International Criminal Court on Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian territories. The formal accession was marked with a ceremony at The Hague, in the Netherlands, where the court is based. The Palestinians signed the ICC\'s founding Rome Statute in January, when they also accepted its jurisdiction over alleged crimes committed "in the occupied Palestinian territory, including East Jerusalem, since June 13, 2014." Later that month, the ICC opened a preliminary examination into the situation in Palestinian territories, paving the way for possible war crimes investigations against Israelis. As members of the court, Palestinians may be subject to counter-charges as well. Israel and the United States, neither of which is an ICC member, opposed the Palestinians\' efforts to join the body. But Palestinian Foreign Minister Riad al-Malki, speaking at Wednesday\'s ceremony, said it was a move toward greater justice. "As Palestine formally becomes a State Party to the Rome Statute today, the world is also a step closer to ending a long era of impunity and injustice," he said, according to an ICC news release. "Indeed, today brings us closer to our shared goals of justice and peace." Judge Kuniko Ozaki, a vice president of the ICC, said acceding to the treaty was just the first step for the Palestinians. "As the Rome Statute today enters into force for the State of Palestine, Palestine acquires all the rights as well as responsibilities that come with being a State Party to the Statute. These are substantive commitments, which cannot be taken lightly," she said. Rights group Human Rights Watch welcomed the development. "Governments seeking to penalize Palestine for joining the ICC should immediately end their pressure, and countries that support universal acceptance of the court\'s treaty should speak out to welcome its membership," said Balkees Jarrah, international justice counsel for the group. "What\'s objectionable is the attempts to undermine international justice, not Palestine\'s decision to join a treaty to which over 100 countries around the world are members." In January, when the preliminary ICC examination was opened, Israeli Prime Minister Benjamin Netanyahu described it as an outrage, saying the court was overstepping its boundaries. The United States also said it "strongly" disagreed with the court\'s decision. "As we have said repeatedly, we do not believe that Palestine is a state and therefore we do not believe that it is eligible to join the ICC," the State Department said in a statement. It urged the warring sides to resolve their differences through direct negotiations. "We will continue to oppose actions against Israel at the ICC as counterproductive to the cause of peace," it said. But the ICC begs to differ with the definition of a state for its purposes and refers to the territories as "Palestine." While a preliminary examination is not a formal investigation, it allows the court to review evidence and determine whether to investigate suspects on both sides. Prosecutor Fatou Bensouda said her office would "conduct its analysis in full independence and impartiality." The war between Israel and Hamas militants in Gaza last summer left more than 2,000 people dead. The inquiry will include alleged war crimes committed since June. The International Criminal Court was set up in 2002 to prosecute genocide, crimes against humanity and war crimes.', + 'The French prosecutor leading an investigation into the crash of Germanwings Flight 9525 insisted Wednesday that he was not aware of any video footage from on board the plane. Marseille prosecutor Brice Robin told CNN that "so far no videos were used in the crash investigation." He added, "A person who has such a video needs to immediately give it to the investigators." Robin\'s comments follow claims by two magazines, German daily Bild and French Paris Match, of a cell phone video showing the harrowing final seconds from on board Germanwings Flight 9525 as it crashed into the French Alps. All 150 on board were killed. Paris Match and Bild reported that the video was recovered from a phone at the wreckage site. The two publications described the supposed video, but did not post it on their websites. The publications said that they watched the video, which was found by a source close to the investigation. "One can hear cries of \'My God\' in several languages," Paris Match reported. "Metallic banging can also be heard more than three times, perhaps of the pilot trying to open the cockpit door with a heavy object. Towards the end, after a heavy shake, stronger than the others, the screaming intensifies. Then nothing." "It is a very disturbing scene," said Julian Reichelt, editor-in-chief of Bild online. An official with France\'s accident investigation agency, the BEA, said the agency is not aware of any such video. Lt. Col. Jean-Marc Menichini, a French Gendarmerie spokesman in charge of communications on rescue efforts around the Germanwings crash site, told CNN that the reports were "completely wrong" and "unwarranted." Cell phones have been collected at the site, he said, but that they "hadn\'t been exploited yet." Menichini said he believed the cell phones would need to be sent to the Criminal Research Institute in Rosny sous-Bois, near Paris, in order to be analyzed by specialized technicians working hand-in-hand with investigators. But none of the cell phones found so far have been sent to the institute, Menichini said. Asked whether staff involved in the search could have leaked a memory card to the media, Menichini answered with a categorical "no." Reichelt told "Erin Burnett: Outfront" that he had watched the video and stood by the report, saying Bild and Paris Match are "very confident" that the clip is real. He noted that investigators only revealed they\'d recovered cell phones from the crash site after Bild and Paris Match published their reports. "That is something we did not know before. ... Overall we can say many things of the investigation weren\'t revealed by the investigation at the beginning," he said. What was mental state of Germanwings co-pilot? German airline Lufthansa confirmed Tuesday that co-pilot Andreas Lubitz had battled depression years before he took the controls of Germanwings Flight 9525, which he\'s accused of deliberately crashing last week in the French Alps. Lubitz told his Lufthansa flight training school in 2009 that he had a "previous episode of severe depression," the airline said Tuesday. Email correspondence between Lubitz and the school discovered in an internal investigation, Lufthansa said, included medical documents he submitted in connection with resuming his flight training. The announcement indicates that Lufthansa, the parent company of Germanwings, knew of Lubitz\'s battle with depression, allowed him to continue training and ultimately put him in the cockpit. Lufthansa, whose CEO Carsten Spohr previously said Lubitz was 100% fit to fly, described its statement Tuesday as a "swift and seamless clarification" and said it was sharing the information and documents -- including training and medical records -- with public prosecutors. Spohr traveled to the crash site Wednesday, where recovery teams have been working for the past week to recover human remains and plane debris scattered across a steep mountainside. He saw the crisis center set up in Seyne-les-Alpes, laid a wreath in the village of Le Vernet, closer to the crash site, where grieving families have left flowers at a simple stone memorial. Menichini told CNN late Tuesday that no visible human remains were left at the site but recovery teams would keep searching. French President Francois Hollande, speaking Tuesday, said that it should be possible to identify all the victims using DNA analysis by the end of the week, sooner than authorities had previously suggested. In the meantime, the recovery of the victims\' personal belongings will start Wednesday, Menichini said. Among those personal belongings could be more cell phones belonging to the 144 passengers and six crew on board. Check out the latest from our correspondents . The details about Lubitz\'s correspondence with the flight school during his training were among several developments as investigators continued to delve into what caused the crash and Lubitz\'s possible motive for downing the jet. A Lufthansa spokesperson told CNN on Tuesday that Lubitz had a valid medical certificate, had passed all his examinations and "held all the licenses required." Earlier, a spokesman for the prosecutor\'s office in Dusseldorf, Christoph Kumpa, said medical records reveal Lubitz suffered from suicidal tendencies at some point before his aviation career and underwent psychotherapy before he got his pilot\'s license. Kumpa emphasized there\'s no evidence suggesting Lubitz was suicidal or acting aggressively before the crash. Investigators are looking into whether Lubitz feared his medical condition would cause him to lose his pilot\'s license, a European government official briefed on the investigation told CNN on Tuesday. While flying was "a big part of his life," the source said, it\'s only one theory being considered. Another source, a law enforcement official briefed on the investigation, also told CNN that authorities believe the primary motive for Lubitz to bring down the plane was that he feared he would not be allowed to fly because of his medical problems. Lubitz\'s girlfriend told investigators he had seen an eye doctor and a neuropsychologist, both of whom deemed him unfit to work recently and concluded he had psychological issues, the European government official said. But no matter what details emerge about his previous mental health struggles, there\'s more to the story, said Brian Russell, a forensic psychologist. "Psychology can explain why somebody would turn rage inward on themselves about the fact that maybe they weren\'t going to keep doing their job and they\'re upset about that and so they\'re suicidal," he said. "But there is no mental illness that explains why somebody then feels entitled to also take that rage and turn it outward on 149 other people who had nothing to do with the person\'s problems." Germanwings crash compensation: What we know . Who was the captain of Germanwings Flight 9525? CNN\'s Margot Haddad reported from Marseille and Pamela Brown from Dusseldorf, while Laura Smith-Spark wrote from London. CNN\'s Frederik Pleitgen, Pamela Boykoff, Antonia Mortensen, Sandrine Amiel and Anna-Maja Rappard contributed to this report.', + ], + return_tensors="tf", + padding="longest", + truncation=True, + ) + features = self.xsum_1_1_model.get_encoder()(**batch).last_hidden_state + + expected = np.array([[-0.0828, -0.0251, -0.0674], [0.1277, 0.3311, -0.0255], [0.2613, -0.0840, -0.2763]]) + assert np.allclose(features[0, :3, :3].numpy(), expected, atol=1e-3) diff --git a/tests/models/opt/test_tokenization_opt.py b/tests/models/opt/test_tokenization_opt.py new file mode 100644 index 0000000000000..b924b7cb7673b --- /dev/null +++ b/tests/models/opt/test_tokenization_opt.py @@ -0,0 +1,185 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import os +import unittest + +from transformers import OPTTokenizer, OPTTokenizerFast, BatchEncoding +from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES +from transformers.testing_utils import require_tokenizers, require_torch +from transformers.utils import cached_property + +from ...test_tokenization_common import TokenizerTesterMixin, filter_roberta_detectors + + +@require_tokenizers +class TestTokenizationOPT(TokenizerTesterMixin, unittest.TestCase): + tokenizer_class = OPTTokenizer + rust_tokenizer_class = OPTTokenizerFast + test_rust_tokenizer = True + from_pretrained_filter = filter_roberta_detectors + # from_pretrained_kwargs = {'add_prefix_space': True} + + def setUp(self): + super().setUp() + vocab = [ + "l", + "o", + "w", + "e", + "r", + "s", + "t", + "i", + "d", + "n", + "\u0120", + "\u0120l", + "\u0120n", + "\u0120lo", + "\u0120low", + "er", + "\u0120lowest", + "\u0120newer", + "\u0120wider", + "", + ] + vocab_tokens = dict(zip(vocab, range(len(vocab)))) + merges = ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""] + self.special_tokens_map = {"unk_token": ""} + + self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"]) + self.merges_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["merges_file"]) + with open(self.vocab_file, "w", encoding="utf-8") as fp: + fp.write(json.dumps(vocab_tokens) + "\n") + with open(self.merges_file, "w", encoding="utf-8") as fp: + fp.write("\n".join(merges)) + + def get_tokenizer(self, **kwargs): + kwargs.update(self.special_tokens_map) + return self.tokenizer_class.from_pretrained(self.tmpdirname, **kwargs) + + def get_rust_tokenizer(self, **kwargs): + kwargs.update(self.special_tokens_map) + return self.rust_tokenizer_class.from_pretrained(self.tmpdirname, **kwargs) + + def get_input_output_texts(self, tokenizer): + return "lower newer", "lower newer" + + @cached_property + def default_tokenizer(self): + return OPTTokenizer.from_pretrained("facebook/opt-large") + + @cached_property + def default_tokenizer_fast(self): + return OPTTokenizerFast.from_pretrained("facebook/opt-large") + + @require_torch + def test_prepare_batch(self): + src_text = ["A long paragraph for summarization.", "Another paragraph for summarization."] + expected_src_tokens = [0, 250, 251, 17818, 13, 39186, 1938, 4, 2] + + for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]: + batch = tokenizer(src_text, max_length=len(expected_src_tokens), padding=True, return_tensors="pt") + self.assertIsInstance(batch, BatchEncoding) + + self.assertEqual((2, 9), batch.input_ids.shape) + self.assertEqual((2, 9), batch.attention_mask.shape) + result = batch.input_ids.tolist()[0] + self.assertListEqual(expected_src_tokens, result) + # Test that special tokens are reset + + @require_torch + def test_prepare_batch_empty_target_text(self): + src_text = ["A long paragraph for summarization.", "Another paragraph for summarization."] + for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]: + batch = tokenizer(src_text, padding=True, return_tensors="pt") + # check if input_ids are returned and no labels + self.assertIn("input_ids", batch) + self.assertIn("attention_mask", batch) + self.assertNotIn("labels", batch) + self.assertNotIn("decoder_attention_mask", batch) + + @require_torch + def test_as_target_tokenizer_target_length(self): + tgt_text = [ + "Summary of the text.", + "Another summary.", + ] + for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]: + with tokenizer.as_target_tokenizer(): + targets = tokenizer(tgt_text, max_length=32, padding="max_length", return_tensors="pt") + self.assertEqual(32, targets["input_ids"].shape[1]) + + @require_torch + def test_prepare_batch_not_longer_than_maxlen(self): + for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]: + batch = tokenizer( + ["I am a small frog" * 1024, "I am a small frog"], padding=True, truncation=True, return_tensors="pt" + ) + self.assertIsInstance(batch, BatchEncoding) + self.assertEqual(batch.input_ids.shape, (2, 1024)) + + @require_torch + def test_special_tokens(self): + + src_text = ["A long paragraph for summarization."] + tgt_text = [ + "Summary of the text.", + ] + for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]: + inputs = tokenizer(src_text, return_tensors="pt") + with tokenizer.as_target_tokenizer(): + targets = tokenizer(tgt_text, return_tensors="pt") + input_ids = inputs["input_ids"] + labels = targets["input_ids"] + self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()) + self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()) + self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()) + self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()) + + def test_pretokenized_inputs(self): + pass + + def test_embeded_special_tokens(self): + for tokenizer, pretrained_name, kwargs in self.tokenizers_list: + with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"): + tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs) + tokenizer_p = self.tokenizer_class.from_pretrained(pretrained_name, **kwargs) + sentence = "A, AllenNLP sentence." + tokens_r = tokenizer_r.encode_plus(sentence, add_special_tokens=True, return_token_type_ids=True) + tokens_p = tokenizer_p.encode_plus(sentence, add_special_tokens=True, return_token_type_ids=True) + + # token_type_ids should put 0 everywhere + self.assertEqual(sum(tokens_r["token_type_ids"]), sum(tokens_p["token_type_ids"])) + + # attention_mask should put 1 everywhere, so sum over length should be 1 + self.assertEqual( + sum(tokens_r["attention_mask"]) / len(tokens_r["attention_mask"]), + sum(tokens_p["attention_mask"]) / len(tokens_p["attention_mask"]), + ) + + tokens_r_str = tokenizer_r.convert_ids_to_tokens(tokens_r["input_ids"]) + tokens_p_str = tokenizer_p.convert_ids_to_tokens(tokens_p["input_ids"]) + + # Rust correctly handles the space before the mask while python doesnt + self.assertSequenceEqual(tokens_p["input_ids"], [0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]) + self.assertSequenceEqual(tokens_r["input_ids"], [0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]) + + self.assertSequenceEqual( + tokens_p_str, ["", "A", ",", "", "ĠAllen", "N", "LP", "Ġsentence", ".", ""] + ) + self.assertSequenceEqual( + tokens_r_str, ["", "A", ",", "", "ĠAllen", "N", "LP", "Ġsentence", ".", ""] + ) From 9ee623d499a41495de52cc7bc1a16c60c35fa9c6 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Wed, 4 May 2022 15:47:57 +0200 Subject: [PATCH 002/145] Final changes - putting use cache to False --- src/transformers/models/opt/configuration_opt.py | 2 +- src/transformers/models/opt/modeling_opt.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/opt/configuration_opt.py b/src/transformers/models/opt/configuration_opt.py index 7a5ad805dc6a5..e3044e6d66ac3 100644 --- a/src/transformers/models/opt/configuration_opt.py +++ b/src/transformers/models/opt/configuration_opt.py @@ -129,7 +129,7 @@ def __init__( classifier_dropout=0.0, scale_embedding=False, share_input_output_embed=True, - use_cache=True, + use_cache=False, num_labels=3, pad_token_id=1, bos_token_id=0, diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 4548579182706..97571d756c4f3 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -785,6 +785,7 @@ def __init__(self, config: OPTConfig): self.share_input_output_embed = config.share_input_output_embed self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0 self.cross_self_attention = False # TODO add it on the config + self.vocab_size = config.vocab_size self.embed_tokens = nn.Embedding(config.vocab_size, config.embed_dim, self.padding_idx) @@ -931,6 +932,7 @@ def forward( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) use_cache = use_cache if use_cache is not None else self.config.use_cache + return_dict = return_dict if return_dict is not None else self.config.use_return_dict # retrieve input_ids and inputs_embeds @@ -945,6 +947,7 @@ def forward( raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds") # past_key_values_length + past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0 if inputs_embeds is None: @@ -1199,8 +1202,8 @@ class OPTForConditionalGeneration(OPTPretrainedModel): def __init__(self, config: OPTConfig): super().__init__(config) self.model = OPTModel(config) - self.register_buffer("final_logits_bias", torch.zeros((1, self.model.shared.num_embeddings))) - self.lm_head = nn.Linear(config.d_model, self.model.shared.num_embeddings, bias=False) + self.register_buffer("final_logits_bias", torch.zeros((1, self.model.decoder.vocab_size))) + self.lm_head = nn.Linear(config.d_model, self.model.decoder.vocab_size, bias=False) # Initialize weights and apply final processing self.post_init() @@ -1775,7 +1778,7 @@ def forward( return_dict=return_dict, ) - logits = self.lm_head(outputs[0]) + logits = self.lm_head(outputs[0]).permute(1, 0, 2) loss = None if labels is not None: From 0484ca10e74a245cd878166e3384eb51a0d97042 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Wed, 4 May 2022 18:45:32 +0200 Subject: [PATCH 003/145] few changes - remove commented block --- src/transformers/models/opt/modeling_opt.py | 94 --------------------- 1 file changed, 94 deletions(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 97571d756c4f3..0666d0f8451d8 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -277,100 +277,6 @@ def forward( v_proj_weight=self.v_proj.weight, ) - # bsz, tgt_len, _ = hidden_states.size() - - # # get query proj - # query_states = self.q_proj(hidden_states) * self.scaling - # # get key, value proj - # if is_cross_attention and past_key_value is not None: - # # reuse k,v, cross_attentions - # key_states = past_key_value[0] - # value_states = past_key_value[1] - # elif is_cross_attention: - # # cross_attentions - # key_states = self._shape(self.k_proj(key_value_states), -1, bsz) - # value_states = self._shape(self.v_proj(key_value_states), -1, bsz) - # elif past_key_value is not None: - # # reuse k, v, self_attention - # key_states = self._shape(self.k_proj(hidden_states), -1, bsz) - # value_states = self._shape(self.v_proj(hidden_states), -1, bsz) - # key_states = torch.cat([past_key_value[0], key_states], dim=2) - # value_states = torch.cat([past_key_value[1], value_states], dim=2) - # else: - # # self_attention - # key_states = self._shape(self.k_proj(hidden_states), -1, bsz) - # value_states = self._shape(self.v_proj(hidden_states), -1, bsz) - - # if self.is_decoder: - # # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. - # # Further calls to cross_attention layer can then reuse all cross-attention - # # key/value_states (first "if" case) - # # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of - # # all previous decoder key/value_states. Further calls to uni-directional self-attention - # # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) - # # if encoder bi-directional self-attention `past_key_value` is always `None` - # past_key_value = (key_states, value_states) - - # proj_shape = (bsz * self.num_heads, -1, self.head_dim) - # query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) - # key_states = key_states.view(*proj_shape) - # value_states = value_states.view(*proj_shape) - - # src_len = key_states.size(1) - # attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) - - # if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): - # raise ValueError( - # f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is {attn_weights.size()}" - # ) - - # if attention_mask is not None: - # if attention_mask.size() != (bsz, 1, tgt_len, src_len): - # raise ValueError( - # f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" - # ) - # attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask - # attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) - - # attn_weights = nn.functional.softmax(attn_weights, dim=-1) - - # if layer_head_mask is not None: - # if layer_head_mask.size() != (self.num_heads,): - # raise ValueError( - # f"Head mask for a single layer should be of size {(self.num_heads,)}, but is {layer_head_mask.size()}" - # ) - # attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(bsz, self.num_heads, tgt_len, src_len) - # attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) - - # if output_attentions: - # # this operation is a bit awkward, but it's required to - # # make sure that attn_weights keeps its gradient. - # # In order to do so, attn_weights have to be reshaped - # # twice and have to be reused in the following - # attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) - # attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) - # else: - # attn_weights_reshaped = None - - # attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) - - # attn_output = torch.bmm(attn_probs, value_states) - - # if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): - # raise ValueError( - # f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is {attn_output.size()}" - # ) - - # attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim) - # attn_output = attn_output.transpose(1, 2) - - # # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be - # # partitioned aross GPUs when using tensor-parallelism. - # attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim) - - # attn_output = self.out_proj(attn_output) - - # return attn_output + residual, attn_weights_reshaped, past_key_value # Copied from transformers.models.bart.modeling_bart.BartEncoderLayer with Bart->OPT From b931db8efbe7e79c7da2114017fe587265e2007e Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Wed, 4 May 2022 18:49:03 +0200 Subject: [PATCH 004/145] few changes - remove unecessary files --- .../models/opt/tokenization_opt.py | 417 ------------------ .../models/opt/tokenization_opt_fast.py | 303 ------------- 2 files changed, 720 deletions(-) delete mode 100644 src/transformers/models/opt/tokenization_opt.py delete mode 100644 src/transformers/models/opt/tokenization_opt_fast.py diff --git a/src/transformers/models/opt/tokenization_opt.py b/src/transformers/models/opt/tokenization_opt.py deleted file mode 100644 index 3749eef6f6351..0000000000000 --- a/src/transformers/models/opt/tokenization_opt.py +++ /dev/null @@ -1,417 +0,0 @@ -# coding=utf-8 -# Copyright 2020 The Facebook AI Research Team Authors and The HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os -from functools import lru_cache -from typing import List, Optional, Tuple - -import regex as re - -from ...tokenization_utils import AddedToken, PreTrainedTokenizer -from ...utils import logging - - -logger = logging.get_logger(__name__) - - -VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt"} - -# See all OPT models at https://huggingface.co/models?filter=opt -PRETRAINED_VOCAB_FILES_MAP = { - "vocab_file": { - "": "https://huggingface.co//resolve/main/vocab.json", - "facebook/opt-large": "https://huggingface.co/facebook/opt-large/resolve/main/vocab.json", - "facebook/opt-large-mnli": "https://huggingface.co/facebook/opt-large-mnli/resolve/main/vocab.json", - "facebook/opt-large-cnn": "https://huggingface.co/facebook/opt-large-cnn/resolve/main/vocab.json", - "facebook/opt-large-xsum": "https://huggingface.co/facebook/opt-large-xsum/resolve/main/vocab.json", - "yjernite/opt_eli5": "https://huggingface.co/yjernite/opt_eli5/resolve/main/vocab.json", - }, - "merges_file": { - "": "https://huggingface.co//resolve/main/merges.txt", - "facebook/opt-large": "https://huggingface.co/facebook/opt-large/resolve/main/merges.txt", - "facebook/opt-large-mnli": "https://huggingface.co/facebook/opt-large-mnli/resolve/main/merges.txt", - "facebook/opt-large-cnn": "https://huggingface.co/facebook/opt-large-cnn/resolve/main/merges.txt", - "facebook/opt-large-xsum": "https://huggingface.co/facebook/opt-large-xsum/resolve/main/merges.txt", - "yjernite/opt_eli5": "https://huggingface.co/yjernite/opt_eli5/resolve/main/merges.txt", - }, -} - -PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { - "": 1024, - "facebook/opt-large": 1024, - "facebook/opt-large-mnli": 1024, - "facebook/opt-large-cnn": 1024, - "facebook/opt-large-xsum": 1024, - "yjernite/opt_eli5": 1024, -} - - -@lru_cache() -def bytes_to_unicode(): - """ - Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control - characters the bpe code barfs on. - - The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab - if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for - decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup - tables between utf-8 bytes and unicode strings. - """ - bs = ( - list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1)) - ) - cs = bs[:] - n = 0 - for b in range(2**8): - if b not in bs: - bs.append(b) - cs.append(2**8 + n) - n += 1 - cs = [chr(n) for n in cs] - return dict(zip(bs, cs)) - - -def get_pairs(word): - """ - Return set of symbol pairs in a word. - - Word is represented as tuple of symbols (symbols being variable-length strings). - """ - pairs = set() - prev_char = word[0] - for char in word[1:]: - pairs.add((prev_char, char)) - prev_char = char - return pairs - - -class OPTTokenizer(PreTrainedTokenizer): - """ - Constructs a OPT tokenizer, which is smilar to the ROBERTa tokenizer, using byte-level Byte-Pair-Encoding. - - This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will - be encoded differently whether it is at the beginning of the sentence (without space) or not: - - ``` - >>> from transformers import OPTTokenizer - >>> tokenizer = OPTTokenizer.from_pretrained("") - >>> tokenizer("Hello world")['input_ids'] - [0, 31414, 232, 2] - >>> tokenizer(" Hello world")['input_ids'] - [0, 20920, 232, 2] - ``` - - You can get around that behavior by passing `add_prefix_space=True` when instantiating this tokenizer or when you - call it on some text, but since the model was not pretrained this way, it might yield a decrease in performance. - - - - When used with `is_split_into_words=True`, this tokenizer will add a space before each word (even the first one). - - - - This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to - this superclass for more information regarding those methods. - - Args: - vocab_file (`str`): - Path to the vocabulary file. - merges_file (`str`): - Path to the merges file. - errors (`str`, *optional*, defaults to `"replace"`): - Paradigm to follow when decoding bytes to UTF-8. See - [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information. - bos_token (`str`, *optional*, defaults to `""`): - The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token. - - - - When building a sequence using special tokens, this is not the token that is used for the beginning of - sequence. The token used is the `cls_token`. - - - - eos_token (`str`, *optional*, defaults to `""`): - The end of sequence token. - - - - When building a sequence using special tokens, this is not the token that is used for the end of sequence. - The token used is the `sep_token`. - - - - sep_token (`str`, *optional*, defaults to `""`): - The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for - sequence classification or for a text and a question for question answering. It is also used as the last - token of a sequence built with special tokens. - cls_token (`str`, *optional*, defaults to `""`): - The classifier token which is used when doing sequence classification (classification of the whole sequence - instead of per-token classification). It is the first token of the sequence when built with special tokens. - unk_token (`str`, *optional*, defaults to `""`): - The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this - token instead. - pad_token (`str`, *optional*, defaults to `""`): - The token used for padding, for example when batching sequences of different lengths. - mask_token (`str`, *optional*, defaults to `""`): - The token used for masking values. This is the token used when training this model with masked language - modeling. This is the token which the model will try to predict. - add_prefix_space (`bool`, *optional*, defaults to `False`): - Whether or not to add an initial space to the input. This allows to treat the leading word just as any - other word. (OPT tokenizer detect beginning of words by the preceding space). - """ - - vocab_files_names = VOCAB_FILES_NAMES - pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP - max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES - model_input_names = ["input_ids", "attention_mask"] - - def __init__( - self, - vocab_file, - merges_file, - errors="replace", - bos_token="", - eos_token="", - sep_token="", - cls_token="", - unk_token="", - pad_token="", - mask_token="", - add_prefix_space=False, - **kwargs - ): - bos_token = AddedToken(bos_token, lstrip=False, rstrip=False) if isinstance(bos_token, str) else bos_token - eos_token = AddedToken(eos_token, lstrip=False, rstrip=False) if isinstance(eos_token, str) else eos_token - sep_token = AddedToken(sep_token, lstrip=False, rstrip=False) if isinstance(sep_token, str) else sep_token - cls_token = AddedToken(cls_token, lstrip=False, rstrip=False) if isinstance(cls_token, str) else cls_token - unk_token = AddedToken(unk_token, lstrip=False, rstrip=False) if isinstance(unk_token, str) else unk_token - pad_token = AddedToken(pad_token, lstrip=False, rstrip=False) if isinstance(pad_token, str) else pad_token - - # Mask token behave like a normal word, i.e. include the space before it - mask_token = AddedToken(mask_token, lstrip=True, rstrip=False) if isinstance(mask_token, str) else mask_token - - super().__init__( - errors=errors, - bos_token=bos_token, - eos_token=eos_token, - unk_token=unk_token, - sep_token=sep_token, - cls_token=cls_token, - pad_token=pad_token, - mask_token=mask_token, - add_prefix_space=add_prefix_space, - **kwargs, - ) - - with open(vocab_file, encoding="utf-8") as vocab_handle: - self.encoder = json.load(vocab_handle) - self.decoder = {v: k for k, v in self.encoder.items()} - self.errors = errors # how to handle errors in decoding - self.byte_encoder = bytes_to_unicode() - self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} - with open(merges_file, encoding="utf-8") as merges_handle: - bpe_merges = merges_handle.read().split("\n")[1:-1] - bpe_merges = [tuple(merge.split()) for merge in bpe_merges] - self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges)))) - self.cache = {} - self.add_prefix_space = add_prefix_space - - # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions - self.pat = re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""") - - @property - def vocab_size(self): - return len(self.encoder) - - def get_vocab(self): - return dict(self.encoder, **self.added_tokens_encoder) - - def bpe(self, token): - if token in self.cache: - return self.cache[token] - word = tuple(token) - pairs = get_pairs(word) - - if not pairs: - return token - - while True: - bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf"))) - if bigram not in self.bpe_ranks: - break - first, second = bigram - new_word = [] - i = 0 - while i < len(word): - try: - j = word.index(first, i) - except ValueError: - new_word.extend(word[i:]) - break - else: - new_word.extend(word[i:j]) - i = j - - if word[i] == first and i < len(word) - 1 and word[i + 1] == second: - new_word.append(first + second) - i += 2 - else: - new_word.append(word[i]) - i += 1 - new_word = tuple(new_word) - word = new_word - if len(word) == 1: - break - else: - pairs = get_pairs(word) - word = " ".join(word) - self.cache[token] = word - return word - - def _tokenize(self, text): - """Tokenize a string.""" - bpe_tokens = [] - for token in re.findall(self.pat, text): - token = "".join( - self.byte_encoder[b] for b in token.encode("utf-8") - ) # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case) - bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(" ")) - return bpe_tokens - - def _convert_token_to_id(self, token): - """Converts a token (str) in an id using the vocab.""" - return self.encoder.get(token, self.encoder.get(self.unk_token)) - - def _convert_id_to_token(self, index): - """Converts an index (integer) in a token (str) using the vocab.""" - return self.decoder.get(index) - - def convert_tokens_to_string(self, tokens): - """Converts a sequence of tokens (string) in a single string.""" - text = "".join(tokens) - text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors=self.errors) - return text - - def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]: - if not os.path.isdir(save_directory): - logger.error(f"Vocabulary path ({save_directory}) should be a directory") - return - vocab_file = os.path.join( - save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] - ) - merge_file = os.path.join( - save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"] - ) - - with open(vocab_file, "w", encoding="utf-8") as f: - f.write(json.dumps(self.encoder, ensure_ascii=False)) - - index = 0 - with open(merge_file, "w", encoding="utf-8") as writer: - writer.write("#version: 0.2\n") - for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]): - if index != token_index: - logger.warning( - f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive." - " Please check that the tokenizer is not corrupted!" - ) - index = token_index - writer.write(" ".join(bpe_tokens) + "\n") - index += 1 - - return vocab_file, merge_file - - def build_inputs_with_special_tokens( - self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None - ) -> List[int]: - """ - Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and - adding special tokens. A OPT sequence has the following format: - - - single sequence: ` X ` - - pair of sequences: ` A B ` - - Args: - token_ids_0 (`List[int]`): - List of IDs to which the special tokens will be added. - token_ids_1 (`List[int]`, *optional*): - Optional second list of IDs for sequence pairs. - - Returns: - `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens. - """ - if token_ids_1 is None: - return [self.cls_token_id] + token_ids_0 + [self.sep_token_id] - cls = [self.cls_token_id] - sep = [self.sep_token_id] - return cls + token_ids_0 + sep + sep + token_ids_1 + sep - - def get_special_tokens_mask( - self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False - ) -> List[int]: - """ - Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding - special tokens using the tokenizer `prepare_for_model` method. - - Args: - token_ids_0 (`List[int]`): - List of IDs. - token_ids_1 (`List[int]`, *optional*): - Optional second list of IDs for sequence pairs. - already_has_special_tokens (`bool`, *optional*, defaults to `False`): - Whether or not the token list is already formatted with special tokens for the model. - - Returns: - `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. - """ - if already_has_special_tokens: - return super().get_special_tokens_mask( - token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True - ) - - if token_ids_1 is None: - return [1] + ([0] * len(token_ids_0)) + [1] - return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1] - - def create_token_type_ids_from_sequences( - self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None - ) -> List[int]: - """ - Create a mask from the two sequences passed to be used in a sequence-pair classification task. OPT does not - make use of token type ids, therefore a list of zeros is returned. - - Args: - token_ids_0 (`List[int]`): - List of IDs. - token_ids_1 (`List[int]`, *optional*): - Optional second list of IDs for sequence pairs. - - Returns: - `List[int]`: List of zeros. - """ - sep = [self.sep_token_id] - cls = [self.cls_token_id] - - if token_ids_1 is None: - return len(cls + token_ids_0 + sep) * [0] - return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0] - - def prepare_for_tokenization(self, text, is_split_into_words=False, **kwargs): - add_prefix_space = kwargs.pop("add_prefix_space", self.add_prefix_space) - if (is_split_into_words or add_prefix_space) and (len(text) > 0 and not text[0].isspace()): - text = " " + text - return (text, kwargs) diff --git a/src/transformers/models/opt/tokenization_opt_fast.py b/src/transformers/models/opt/tokenization_opt_fast.py deleted file mode 100644 index 429baa552cc29..0000000000000 --- a/src/transformers/models/opt/tokenization_opt_fast.py +++ /dev/null @@ -1,303 +0,0 @@ -# coding=utf-8 -# Copyright 2020 The Facebook AI Research Team Authors and The HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -from typing import List, Optional, Tuple - -from tokenizers import pre_tokenizers, processors - -from ...tokenization_utils_base import AddedToken, BatchEncoding -from ...tokenization_utils_fast import PreTrainedTokenizerFast -from ...utils import logging -from .tokenization_opt import OPTTokenizer - - -logger = logging.get_logger(__name__) - - -VOCAB_FILES_NAMES = {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"} - -# See all OPT models at https://huggingface.co/models?filter=opt -PRETRAINED_VOCAB_FILES_MAP = { - "vocab_file": { - "": "https://huggingface.co//resolve/main/vocab.json", - "facebook/opt-large": "https://huggingface.co/facebook/opt-large/resolve/main/vocab.json", - "facebook/opt-large-mnli": "https://huggingface.co/facebook/opt-large-mnli/resolve/main/vocab.json", - "facebook/opt-large-cnn": "https://huggingface.co/facebook/opt-large-cnn/resolve/main/vocab.json", - "facebook/opt-large-xsum": "https://huggingface.co/facebook/opt-large-xsum/resolve/main/vocab.json", - "yjernite/opt_eli5": "https://huggingface.co/yjernite/opt_eli5/resolve/main/vocab.json", - }, - "merges_file": { - "": "https://huggingface.co//resolve/main/merges.txt", - "facebook/opt-large": "https://huggingface.co/facebook/opt-large/resolve/main/merges.txt", - "facebook/opt-large-mnli": "https://huggingface.co/facebook/opt-large-mnli/resolve/main/merges.txt", - "facebook/opt-large-cnn": "https://huggingface.co/facebook/opt-large-cnn/resolve/main/merges.txt", - "facebook/opt-large-xsum": "https://huggingface.co/facebook/opt-large-xsum/resolve/main/merges.txt", - "yjernite/opt_eli5": "https://huggingface.co/yjernite/opt_eli5/resolve/main/merges.txt", - }, - "tokenizer_file": { - "": "https://huggingface.co//resolve/main/tokenizer.json", - "facebook/opt-large": "https://huggingface.co/facebook/opt-large/resolve/main/tokenizer.json", - "facebook/opt-large-mnli": "https://huggingface.co/facebook/opt-large-mnli/resolve/main/tokenizer.json", - "facebook/opt-large-cnn": "https://huggingface.co/facebook/opt-large-cnn/resolve/main/tokenizer.json", - "facebook/opt-large-xsum": "https://huggingface.co/facebook/opt-large-xsum/resolve/main/tokenizer.json", - "yjernite/opt_eli5": "https://huggingface.co/yjernite/opt_eli5/resolve/main/tokenizer.json", - }, -} - -PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { - "": 1024, - "facebook/opt-large": 1024, - "facebook/opt-large-mnli": 1024, - "facebook/opt-large-cnn": 1024, - "facebook/opt-large-xsum": 1024, - "yjernite/opt_eli5": 1024, -} - - -class OPTTokenizerFast(PreTrainedTokenizerFast): - r""" - Construct a "fast" OPT tokenizer (backed by HuggingFace's *tokenizers* library), derived from the GPT-2 tokenizer, - using byte-level Byte-Pair-Encoding. - - This tokenizer has been trained to treat spaces like parts of the tokens (a bit like sentencepiece) so a word will - be encoded differently whether it is at the beginning of the sentence (without space) or not: - - ``` - >>> from transformers import OPTTokenizerFast - >>> tokenizer = OPTTokenizerFast.from_pretrained("") - >>> tokenizer("Hello world")['input_ids'] - [0, 31414, 232, 2] - >>> tokenizer(" Hello world")['input_ids'] - [0, 20920, 232, 2] - ``` - - You can get around that behavior by passing `add_prefix_space=True` when instantiating this tokenizer or when you - call it on some text, but since the model was not pretrained this way, it might yield a decrease in performance. - - - - When used with `is_split_into_words=True`, this tokenizer needs to be instantiated with `add_prefix_space=True`. - - - - This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should - refer to this superclass for more information regarding those methods. - - Args: - vocab_file (`str`): - Path to the vocabulary file. - merges_file (`str`): - Path to the merges file. - errors (`str`, *optional*, defaults to `"replace"`): - Paradigm to follow when decoding bytes to UTF-8. See - [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information. - bos_token (`str`, *optional*, defaults to `""`): - The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token. - - - - When building a sequence using special tokens, this is not the token that is used for the beginning of - sequence. The token used is the `cls_token`. - - - - eos_token (`str`, *optional*, defaults to `""`): - The end of sequence token. - - - - When building a sequence using special tokens, this is not the token that is used for the end of sequence. - The token used is the `sep_token`. - - - - sep_token (`str`, *optional*, defaults to `""`): - The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for - sequence classification or for a text and a question for question answering. It is also used as the last - token of a sequence built with special tokens. - cls_token (`str`, *optional*, defaults to `""`): - The classifier token which is used when doing sequence classification (classification of the whole sequence - instead of per-token classification). It is the first token of the sequence when built with special tokens. - unk_token (`str`, *optional*, defaults to `""`): - The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this - token instead. - pad_token (`str`, *optional*, defaults to `""`): - The token used for padding, for example when batching sequences of different lengths. - mask_token (`str`, *optional*, defaults to `""`): - The token used for masking values. This is the token used when training this model with masked language - modeling. This is the token which the model will try to predict. - add_prefix_space (`bool`, *optional*, defaults to `False`): - Whether or not to add an initial space to the input. This allows to treat the leading word just as any - other word. (OPT tokenizer detect beginning of words by the preceding space). - trim_offsets (`bool`, *optional*, defaults to `True`): - Whether the post processing step should trim offsets to avoid including whitespaces. - """ - vocab_files_names = VOCAB_FILES_NAMES - pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP - max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES - model_input_names = ["input_ids", "attention_mask"] - slow_tokenizer_class = OPTTokenizer - - def __init__( - self, - vocab_file=None, - merges_file=None, - tokenizer_file=None, - errors="replace", - bos_token="", - eos_token="", - sep_token="", - cls_token="", - unk_token="", - pad_token="", - mask_token="", - add_prefix_space=False, - trim_offsets=True, - **kwargs - ): - super().__init__( - vocab_file, - merges_file, - tokenizer_file=tokenizer_file, - errors=errors, - bos_token=bos_token, - eos_token=eos_token, - sep_token=sep_token, - cls_token=cls_token, - unk_token=unk_token, - pad_token=pad_token, - mask_token=mask_token, - add_prefix_space=add_prefix_space, - trim_offsets=trim_offsets, - **kwargs, - ) - - pre_tok_state = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()) - if pre_tok_state.get("add_prefix_space", add_prefix_space) != add_prefix_space: - pre_tok_class = getattr(pre_tokenizers, pre_tok_state.pop("type")) - pre_tok_state["add_prefix_space"] = add_prefix_space - self.backend_tokenizer.pre_tokenizer = pre_tok_class(**pre_tok_state) - - self.add_prefix_space = add_prefix_space - - # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__` - tokenizer_component = "post_processor" - tokenizer_component_instance = getattr(self.backend_tokenizer, tokenizer_component, None) - if tokenizer_component_instance: - state = json.loads(tokenizer_component_instance.__getstate__()) - - # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class` - if "sep" in state: - state["sep"] = tuple(state["sep"]) - if "cls" in state: - state["cls"] = tuple(state["cls"]) - - changes_to_apply = False - - if state.get("add_prefix_space", add_prefix_space) != add_prefix_space: - state["add_prefix_space"] = add_prefix_space - changes_to_apply = True - - if state.get("trim_offsets", trim_offsets) != trim_offsets: - state["trim_offsets"] = trim_offsets - changes_to_apply = True - - if changes_to_apply: - component_class = getattr(processors, state.pop("type")) - new_value = component_class(**state) - setattr(self.backend_tokenizer, tokenizer_component, new_value) - - @property - def mask_token(self) -> str: - """ - `str`: Mask token, to use when training a model with masked-language modeling. Log an error if used while not - having been set. - - OPT tokenizer has a special mask token to be usable in the fill-mask pipeline. The mask token will greedily - comprise the space before the **. - """ - if self._mask_token is None and self.verbose: - logger.error("Using mask_token, but it is not set yet.") - return None - return str(self._mask_token) - - @mask_token.setter - def mask_token(self, value): - """ - Overriding the default behavior of the mask token to have it eat the space before it. - - This is needed to preserve backward compatibility with all the previously used models based on OPT. - """ - # Mask token behave like a normal word, i.e. include the space before it - # So we set lstrip to True - value = AddedToken(value, lstrip=True, rstrip=False) if isinstance(value, str) else value - self._mask_token = value - - def _batch_encode_plus(self, *args, **kwargs) -> BatchEncoding: - is_split_into_words = kwargs.get("is_split_into_words", False) - - if is_split_into_words and not self.add_prefix_space: - raise ValueError( - f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True " - "to use it with pretokenized inputs." - ) - - return super()._batch_encode_plus(*args, **kwargs) - - def _encode_plus(self, *args, **kwargs) -> BatchEncoding: - is_split_into_words = kwargs.get("is_split_into_words", False) - - if is_split_into_words and not self.add_prefix_space: - raise ValueError( - f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True " - "to use it with pretokenized inputs." - ) - - return super()._encode_plus(*args, **kwargs) - - def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]: - files = self._tokenizer.model.save(save_directory, name=filename_prefix) - return tuple(files) - - def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): - output = [self.bos_token_id] + token_ids_0 + [self.eos_token_id] - if token_ids_1 is None: - return output - - return output + [self.eos_token_id] + token_ids_1 + [self.eos_token_id] - - def create_token_type_ids_from_sequences( - self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None - ) -> List[int]: - """ - Create a mask from the two sequences passed to be used in a sequence-pair classification task. OPT does not - make use of token type ids, therefore a list of zeros is returned. - - Args: - token_ids_0 (`List[int]`): - List of IDs. - token_ids_1 (`List[int]`, *optional*): - Optional second list of IDs for sequence pairs. - - Returns: - `List[int]`: List of zeros. - """ - sep = [self.sep_token_id] - cls = [self.cls_token_id] - - if token_ids_1 is None: - return len(cls + token_ids_0 + sep) * [0] - return len(cls + token_ids_0 + sep + sep + token_ids_1 + sep) * [0] From 681dfc501be00417bbee1051f95b93a1d0be3483 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Wed, 4 May 2022 18:57:25 +0200 Subject: [PATCH 005/145] fix style issues --- src/transformers/__init__.py | 98 +++++++------- src/transformers/models/__init__.py | 2 +- .../models/opt/configuration_opt.py | 7 +- ..._original_pytorch_checkpoint_to_pytorch.py | 10 +- .../models/opt/modeling_flax_opt.py | 8 +- src/transformers/models/opt/modeling_opt.py | 121 ++++++++---------- tests/models/opt/test_embeddings_opt.py | 23 +--- tests/models/opt/test_tokenization_opt.py | 2 +- 8 files changed, 117 insertions(+), 154 deletions(-) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index e43ba28e9962d..c016bb0902554 100755 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -131,7 +131,6 @@ "AutoTokenizer", ], "models.bart": ["BartConfig", "BartTokenizer"], - "models.opt": ["OPTConfig", "OPTTokenizer"], "models.barthez": [], "models.bartpho": [], "models.beit": ["BEIT_PRETRAINED_CONFIG_ARCHIVE_MAP", "BeitConfig"], @@ -239,6 +238,7 @@ "NystromformerConfig", ], "models.openai": ["OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP", "OpenAIGPTConfig", "OpenAIGPTTokenizer"], + "models.opt": ["OPTConfig", "OPTTokenizer"], "models.pegasus": ["PEGASUS_PRETRAINED_CONFIG_ARCHIVE_MAP", "PegasusConfig", "PegasusTokenizer"], "models.perceiver": ["PERCEIVER_PRETRAINED_CONFIG_ARCHIVE_MAP", "PerceiverConfig", "PerceiverTokenizer"], "models.phobert": ["PhobertTokenizer"], @@ -452,7 +452,6 @@ # Fast tokenizers _import_structure["models.albert"].append("AlbertTokenizerFast") _import_structure["models.bart"].append("BartTokenizerFast") - _import_structure["models.opt"].append("OPTTokenizerFast") _import_structure["models.barthez"].append("BarthezTokenizerFast") _import_structure["models.bert"].append("BertTokenizerFast") _import_structure["models.big_bird"].append("BigBirdTokenizerFast") @@ -485,6 +484,7 @@ _import_structure["models.mpnet"].append("MPNetTokenizerFast") _import_structure["models.mt5"].append("MT5TokenizerFast") _import_structure["models.openai"].append("OpenAIGPTTokenizerFast") + _import_structure["models.opt"].append("OPTTokenizerFast") _import_structure["models.pegasus"].append("PegasusTokenizerFast") _import_structure["models.realm"].append("RealmTokenizerFast") _import_structure["models.reformer"].append("ReformerTokenizerFast") @@ -733,18 +733,6 @@ "PretrainedBartModel", ] ) - _import_structure["models.opt"].extend( - [ - "OPT_PRETRAINED_MODEL_ARCHIVE_LIST", - "OPTForCausalLM", - "OPTForConditionalGeneration", - "OPTForQuestionAnswering", - "OPTForSequenceClassification", - "OPTModel", - "OPTPretrainedModel", - "PretrainedOPTModel", - ] - ) _import_structure["models.beit"].extend( [ "BEIT_PRETRAINED_MODEL_ARCHIVE_LIST", @@ -1283,6 +1271,18 @@ "load_tf_weights_in_openai_gpt", ] ) + _import_structure["models.opt"].extend( + [ + "OPT_PRETRAINED_MODEL_ARCHIVE_LIST", + "OPTForCausalLM", + "OPTForConditionalGeneration", + "OPTForQuestionAnswering", + "OPTForSequenceClassification", + "OPTModel", + "OPTPretrainedModel", + "PretrainedOPTModel", + ] + ) _import_structure["models.pegasus"].extend( ["PegasusForCausalLM", "PegasusForConditionalGeneration", "PegasusModel", "PegasusPreTrainedModel"] ) @@ -1819,7 +1819,6 @@ ] ) _import_structure["models.bart"].extend(["TFBartForConditionalGeneration", "TFBartModel", "TFBartPretrainedModel"]) - _import_structure["models.opt"].extend(["TFOPTForConditionalGeneration", "TFOPTModel", "TFOPTPretrainedModel"]) _import_structure["models.bert"].extend( [ "TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST", @@ -2088,6 +2087,7 @@ "TFOpenAIGPTPreTrainedModel", ] ) + _import_structure["models.opt"].extend(["TFOPTForConditionalGeneration", "TFOPTModel", "TFOPTPretrainedModel"]) _import_structure["models.pegasus"].extend( ["TFPegasusForConditionalGeneration", "TFPegasusModel", "TFPegasusPreTrainedModel"] ) @@ -2318,18 +2318,6 @@ "FlaxBartPreTrainedModel", ] ) - _import_structure["models.opt"].extend( - [ - "FlaxOPTDecoderPreTrainedModel", - "FlaxOPTForCausalLM", - "FlaxOPTForConditionalGeneration", - "FlaxOPTForQuestionAnswering", - "FlaxOPTForSequenceClassification", - "FlaxOPTModel", - "FlaxOPTPreTrainedModel", - ] - ) - _import_structure["models.beit"].extend( [ "FlaxBeitForImageClassification", @@ -2338,6 +2326,7 @@ "FlaxBeitPreTrainedModel", ] ) + _import_structure["models.bert"].extend( [ "FlaxBertForCausalLM", @@ -2432,6 +2421,17 @@ ] ) _import_structure["models.mt5"].extend(["FlaxMT5ForConditionalGeneration", "FlaxMT5Model"]) + _import_structure["models.opt"].extend( + [ + "FlaxOPTDecoderPreTrainedModel", + "FlaxOPTForCausalLM", + "FlaxOPTForConditionalGeneration", + "FlaxOPTForQuestionAnswering", + "FlaxOPTForSequenceClassification", + "FlaxOPTModel", + "FlaxOPTPreTrainedModel", + ] + ) _import_structure["models.pegasus"].extend( [ "FlaxPegasusForConditionalGeneration", @@ -2577,7 +2577,6 @@ AutoTokenizer, ) from .models.bart import BartConfig, BartTokenizer - from .models.opt import OPTConfig, OPTTokenizer from .models.beit import BEIT_PRETRAINED_CONFIG_ARCHIVE_MAP, BeitConfig from .models.bert import ( BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, @@ -2673,6 +2672,7 @@ from .models.mt5 import MT5Config from .models.nystromformer import NYSTROMFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, NystromformerConfig from .models.openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig, OpenAIGPTTokenizer + from .models.opt import OPTConfig, OPTTokenizer from .models.pegasus import PEGASUS_PRETRAINED_CONFIG_ARCHIVE_MAP, PegasusConfig, PegasusTokenizer from .models.perceiver import PERCEIVER_PRETRAINED_CONFIG_ARCHIVE_MAP, PerceiverConfig, PerceiverTokenizer from .models.phobert import PhobertTokenizer @@ -2867,7 +2867,6 @@ if is_tokenizers_available(): from .models.albert import AlbertTokenizerFast from .models.bart import BartTokenizerFast - from .models.opt import OPTTokenizerFast from .models.barthez import BarthezTokenizerFast from .models.bert import BertTokenizerFast from .models.big_bird import BigBirdTokenizerFast @@ -2898,6 +2897,7 @@ from .models.mpnet import MPNetTokenizerFast from .models.mt5 import MT5TokenizerFast from .models.openai import OpenAIGPTTokenizerFast + from .models.opt import OPTTokenizerFast from .models.pegasus import PegasusTokenizerFast from .models.realm import RealmTokenizerFast from .models.reformer import ReformerTokenizerFast @@ -3097,16 +3097,6 @@ BartPretrainedModel, PretrainedBartModel, ) - from .models.opt import ( - OPT_PRETRAINED_MODEL_ARCHIVE_LIST, - OPTForCausalLM, - OPTForConditionalGeneration, - OPTForQuestionAnswering, - OPTForSequenceClassification, - OPTModel, - OPTPretrainedModel, - PretrainedOPTModel, - ) from .models.beit import ( BEIT_PRETRAINED_MODEL_ARCHIVE_LIST, BeitForImageClassification, @@ -3553,6 +3543,16 @@ OpenAIGPTPreTrainedModel, load_tf_weights_in_openai_gpt, ) + from .models.opt import ( + OPT_PRETRAINED_MODEL_ARCHIVE_LIST, + OPTForCausalLM, + OPTForConditionalGeneration, + OPTForQuestionAnswering, + OPTForSequenceClassification, + OPTModel, + OPTPretrainedModel, + PretrainedOPTModel, + ) from .models.pegasus import ( PegasusForCausalLM, PegasusForConditionalGeneration, @@ -4006,7 +4006,6 @@ TFAutoModelWithLMHead, ) from .models.bart import TFBartForConditionalGeneration, TFBartModel, TFBartPretrainedModel - from .models.opt import TFOPTForConditionalGeneration, TFOPTModel, TFOPTPretrainedModel from .models.bert import ( TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST, TFBertEmbeddings, @@ -4220,6 +4219,7 @@ TFOpenAIGPTModel, TFOpenAIGPTPreTrainedModel, ) + from .models.opt import TFOPTForConditionalGeneration, TFOPTModel, TFOPTPretrainedModel from .models.pegasus import TFPegasusForConditionalGeneration, TFPegasusModel, TFPegasusPreTrainedModel from .models.rag import TFRagModel, TFRagPreTrainedModel, TFRagSequenceForGeneration, TFRagTokenForGeneration from .models.rembert import ( @@ -4399,15 +4399,6 @@ FlaxBartModel, FlaxBartPreTrainedModel, ) - from .models.opt import ( - FlaxOPTDecoderPreTrainedModel, - FlaxOPTForCausalLM, - FlaxOPTForConditionalGeneration, - FlaxOPTForQuestionAnswering, - FlaxOPTForSequenceClassification, - FlaxOPTModel, - FlaxOPTPreTrainedModel, - ) from .models.beit import ( FlaxBeitForImageClassification, FlaxBeitForMaskedImageModeling, @@ -4488,6 +4479,15 @@ FlaxMBartPreTrainedModel, ) from .models.mt5 import FlaxMT5ForConditionalGeneration, FlaxMT5Model + from .models.opt import ( + FlaxOPTDecoderPreTrainedModel, + FlaxOPTForCausalLM, + FlaxOPTForConditionalGeneration, + FlaxOPTForQuestionAnswering, + FlaxOPTForSequenceClassification, + FlaxOPTModel, + FlaxOPTPreTrainedModel, + ) from .models.pegasus import FlaxPegasusForConditionalGeneration, FlaxPegasusModel, FlaxPegasusPreTrainedModel from .models.roberta import ( FlaxRobertaForCausalLM, diff --git a/src/transformers/models/__init__.py b/src/transformers/models/__init__.py index 93922f7d901fa..380e043969f46 100644 --- a/src/transformers/models/__init__.py +++ b/src/transformers/models/__init__.py @@ -20,7 +20,6 @@ albert, auto, bart, - opt, barthez, bartpho, beit, @@ -87,6 +86,7 @@ mt5, nystromformer, openai, + opt, pegasus, perceiver, phobert, diff --git a/src/transformers/models/opt/configuration_opt.py b/src/transformers/models/opt/configuration_opt.py index e3044e6d66ac3..1629a596c9cc5 100644 --- a/src/transformers/models/opt/configuration_opt.py +++ b/src/transformers/models/opt/configuration_opt.py @@ -31,11 +31,10 @@ } - class OPTConfig(PretrainedConfig): r""" - This is the configuration class to store the configuration of a [`OPTModel`]. It is used to instantiate a OPT - model according to the specified arguments, defining the model architecture. Instantiating a configuration with the + This is the configuration class to store the configuration of a [`OPTModel`]. It is used to instantiate a OPT model + according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the OPT [facebook/opt-large](https://huggingface.co/facebook/opt-large) architecture. @@ -113,7 +112,7 @@ class OPTConfig(PretrainedConfig): def __init__( self, - vocab_size=50272, # TODO check the real value + vocab_size=50272, # TODO check the real value max_position_embeddings=2048, decoder_layers=24, decoder_ffn_dim=4096, diff --git a/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py index 088592f2c9ea0..1ce87d3b72962 100644 --- a/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py @@ -16,21 +16,13 @@ import argparse -import os from pathlib import Path # import fairseq import torch -from packaging import version from torch import nn -from transformers import ( - OPTConfig, - OPTForConditionalGeneration, - OPTForSequenceClassification, - OPTModel, - OPTTokenizer, -) +from transformers import OPTConfig, OPTForSequenceClassification, OPTModel from transformers.utils import logging diff --git a/src/transformers/models/opt/modeling_flax_opt.py b/src/transformers/models/opt/modeling_flax_opt.py index 296f193d92dca..3662620b0fd0f 100644 --- a/src/transformers/models/opt/modeling_flax_opt.py +++ b/src/transformers/models/opt/modeling_flax_opt.py @@ -1735,8 +1735,8 @@ def __call__( @add_start_docstrings( """ - OPT Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layer on top of the hidden-states output to compute `span start logits` and `span end logits`). + OPT Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layer + on top of the hidden-states output to compute `span start logits` and `span end logits`). """, OPT_START_DOCSTRING, ) @@ -1976,8 +1976,8 @@ def __call__( @add_start_docstrings( """ - OPT Decoder Model with a language modeling head on top (linear layer with weights tied to the input embeddings) - e.g for autoregressive tasks. + OPT Decoder Model with a language modeling head on top (linear layer with weights tied to the input embeddings) e.g + for autoregressive tasks. """, OPT_START_DOCSTRING, ) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 0666d0f8451d8..01f5bdfba0080 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -17,13 +17,12 @@ import math import random import warnings -from typing import List, Optional, Tuple, Union, Dict +from typing import Dict, List, Optional, Tuple, Union import torch -import torch.utils.checkpoint import torch.nn.functional as F -from torch import Tensor -from torch import nn +import torch.utils.checkpoint +from torch import Tensor, nn from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss from ...activations import ACT2FN @@ -74,7 +73,6 @@ ] - def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int): """ Shift input ids one token to the right. @@ -123,8 +121,7 @@ def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] # Copied from transformers.models.bart.modeling_bart.BartLearnedPositionalEmbedding with Bart->OPT # class OPTLearnedPositionalEmbedding(nn.Embedding): # """ -# This module learns positional embeddings up to a fixed maximum size. -# """ +# This module learns positional embeddings up to a fixed maximum size. #""" # def __init__(self, num_embeddings: int, embedding_dim: int): # # OPT is set up so that if padding_idx is specified then offset the embedding ids by 2 @@ -140,6 +137,7 @@ def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] # ) # return super().forward(positions + self.offset) + def make_positions(tensor, padding_idx: int, onnx_trace: bool = False): """Replace non-padding symbols with their position numbers. @@ -152,15 +150,15 @@ def make_positions(tensor, padding_idx: int, onnx_trace: bool = False): mask = tensor.ne(padding_idx).int() return (torch.cumsum(mask, dim=1).type_as(mask) * mask).long() + padding_idx + class LearnedPositionalEmbedding(nn.Embedding): """ - This module learns positional embeddings up to a fixed maximum size. - Padding ids are ignored by either offsetting based on padding_idx - or by setting padding_idx to None and ensuring that the appropriate - position ids are passed to the forward function. + This module learns positional embeddings up to a fixed maximum size. Padding ids are ignored by either offsetting + based on padding_idx or by setting padding_idx to None and ensuring that the appropriate position ids are passed to + the forward function. """ - def __init__(self, num_embeddings: int, embedding_dim: int, padding_idx: int=1): + def __init__(self, num_embeddings: int, embedding_dim: int, padding_idx: int = 1): super().__init__(num_embeddings, embedding_dim, padding_idx) self.onnx_trace = False if self.padding_idx is not None: @@ -183,9 +181,7 @@ def forward( # padding. if positions is None: - positions = make_positions( - input, self.padding_idx, onnx_trace=self.onnx_trace - ) + positions = make_positions(input, self.padding_idx, onnx_trace=self.onnx_trace) return F.embedding( positions, self.weight, @@ -196,6 +192,7 @@ def forward( self.sparse, ) + # Copied from transformers.models.bart.modeling_bart.BartAttention with Bart->OPT class OPTAttention(nn.Module): """Multi-headed attention from 'Attention Is All You Need' paper""" @@ -207,7 +204,7 @@ def __init__( dropout: float = 0.1, is_decoder: bool = True, bias: bool = True, - add_zero_attn = False + add_zero_attn=False, ): super().__init__() self.embed_dim = embed_dim @@ -246,37 +243,35 @@ def forward( # if key_value_states are provided this layer is used as a cross-attention layer # for the decoder - is_cross_attention = key_value_states is not None - residual = hidden_states # create key padding mask + # HACK : to fix assert torch.equal(attention_mask[0, 0, :, :], attention_mask[-1, 0, :, :]) attention_mask = attention_mask[0, 0, :, :] return F.multi_head_attention_forward( - hidden_states, - hidden_states, - hidden_states, - self.embed_dim, - self.num_heads, - torch.empty([0]), - torch.cat((self.q_proj.bias, self.k_proj.bias, self.v_proj.bias)), - None, - None, - self.add_zero_attn, - self.dropout, - self.out_proj.weight, - self.out_proj.bias, - self.training, - self_attn_padding_mask, - False, - attention_mask, - use_separate_proj_weight=True, - q_proj_weight=self.q_proj.weight, - k_proj_weight=self.k_proj.weight, - v_proj_weight=self.v_proj.weight, - ) - + hidden_states, + hidden_states, + hidden_states, + self.embed_dim, + self.num_heads, + torch.empty([0]), + torch.cat((self.q_proj.bias, self.k_proj.bias, self.v_proj.bias)), + None, + None, + self.add_zero_attn, + self.dropout, + self.out_proj.weight, + self.out_proj.bias, + self.training, + self_attn_padding_mask, + False, + attention_mask, + use_separate_proj_weight=True, + q_proj_weight=self.q_proj.weight, + k_proj_weight=self.k_proj.weight, + v_proj_weight=self.v_proj.weight, + ) # Copied from transformers.models.bart.modeling_bart.BartEncoderLayer with Bart->OPT @@ -403,8 +398,6 @@ def forward( # hidden_states = hidden_states.permute(1, 0, 2) residual = hidden_states - - # Self Attention # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None @@ -421,7 +414,6 @@ def forward( hidden_states = residual + hidden_states hidden_states = self.self_attn_layer_norm(hidden_states) # Cross-Attention Block - cross_attn_present_key_value = None cross_attn_weights = None # Fully Connected @@ -430,7 +422,7 @@ def forward( residual = hidden_states hidden_states = self.fc1(hidden_states) hidden_states = self.activation_fn(hidden_states) - #hidden_states = nn.functional.dropout(hidden_states, p=self.activation_dropout, training=self.training) + # hidden_states = nn.functional.dropout(hidden_states, p=self.activation_dropout, training=self.training) hidden_states = self.fc2(hidden_states) hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) @@ -493,7 +485,7 @@ def _init_weights(self, module): module.weight.data[module.padding_idx].zero_() def _set_gradient_checkpointing(self, module, value=False): - if isinstance(module, (OPTDecoder, OPTEncoder)): + if isinstance(module, (OPTDecoder)): module.gradient_checkpointing = value @property @@ -610,9 +602,9 @@ def __init_subclass__(self): Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also be used by default. - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_inputs`] and - modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information - on the default strategy. + If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_inputs`] and modify + to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the + default strategy. head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: @@ -690,7 +682,7 @@ def __init__(self, config: OPTConfig): self.max_target_positions = config.max_position_embeddings self.share_input_output_embed = config.share_input_output_embed self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0 - self.cross_self_attention = False # TODO add it on the config + self.cross_self_attention = False # TODO add it on the config self.vocab_size = config.vocab_size self.embed_tokens = nn.Embedding(config.vocab_size, config.embed_dim, self.padding_idx) @@ -704,23 +696,14 @@ def __init__(self, config: OPTConfig): ) self.project_out_dim = ( - nn.Linear(config.d_model, config.embed_dim, bias=False) - if config.embed_dim != config.d_model - else None + nn.Linear(config.d_model, config.embed_dim, bias=False) if config.embed_dim != config.d_model else None ) - self.project_in_dim = ( - nn.Linear(config.embed_dim, config.d_model, bias=False) - if config.d_model != config.embed_dim - else None + nn.Linear(config.embed_dim, config.d_model, bias=False) if config.d_model != config.embed_dim else None ) - - self.output_projection = nn.Linear( - config.embed_dim, config.vocab_size, bias=False - ) - + self.output_projection = nn.Linear(config.embed_dim, config.vocab_size, bias=False) self.layers = nn.ModuleList([OPTDecoderLayer(config) for _ in range(config.decoder_layers)]) @@ -838,7 +821,7 @@ def forward( output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states ) use_cache = use_cache if use_cache is not None else self.config.use_cache - + return_dict = return_dict if return_dict is not None else self.config.use_return_dict # retrieve input_ids and inputs_embeds @@ -853,7 +836,7 @@ def forward( raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds") # past_key_values_length - + past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0 if inputs_embeds is None: @@ -863,9 +846,7 @@ def forward( attention_mask, input_shape, inputs_embeds, past_key_values_length ) - if self_attn_padding_mask is None and ( - self.cross_self_attention or input_ids.eq(self.padding_idx).any() - ): + if self_attn_padding_mask is None and (self.cross_self_attention or input_ids.eq(self.padding_idx).any()): self_attn_padding_mask = input_ids.eq(self.padding_idx) # expand encoder attention mask @@ -954,7 +935,7 @@ def custom_forward(*inputs): if encoder_hidden_states is not None: all_cross_attentions += (layer_outputs[2],) - + if self.project_out_dim is not None: hidden_states = self.project_out_dim(hidden_states) @@ -1393,8 +1374,8 @@ def forward( @add_start_docstrings( """ - OPT Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear - layer on top of the hidden-states output to compute `span start logits` and `span end logits`). + OPT Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layer + on top of the hidden-states output to compute `span start logits` and `span end logits`). """, OPT_START_DOCSTRING, ) diff --git a/tests/models/opt/test_embeddings_opt.py b/tests/models/opt/test_embeddings_opt.py index fc179f559060a..8d3f11903edab 100644 --- a/tests/models/opt/test_embeddings_opt.py +++ b/tests/models/opt/test_embeddings_opt.py @@ -16,37 +16,27 @@ import unittest - -from transformers import OPTConfig, is_torch_available, BartTokenizerFast - +from transformers import BartTokenizerFast, is_torch_available if is_torch_available(): import torch - from transformers import ( - AutoModelForSequenceClassification, - OPTForCausalLM, - OPTForConditionalGeneration, - OPTForQuestionAnswering, - OPTForSequenceClassification, - OPTModel, - OPTTokenizer, - pipeline, - ) + from transformers import OPTForCausalLM + class OPTEmbeddingsTest(unittest.TestCase): def setUp(self): super().setUp() self.path_model = "/home/younes/Desktop/Work/data/opt-350-m/" self.path_logits_meta = "/home/younes/Desktop/Work/metaseq-conversion/logits_metaseq.p" - + def test_load_model(self): try: _ = OPTForCausalLM.from_pretrained(self.path_model) except BaseException: self.fail("Failed loading model") - + def test_logits(self): model = OPTForCausalLM.from_pretrained(self.path_model) model = model.eval() @@ -62,5 +52,6 @@ def test_logits(self): logits_meta = torch.load(self.path_logits_meta) assert torch.allclose(logits, logits_meta.permute(1, 0, 2)) + if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/models/opt/test_tokenization_opt.py b/tests/models/opt/test_tokenization_opt.py index b924b7cb7673b..7dc93eb9ca23b 100644 --- a/tests/models/opt/test_tokenization_opt.py +++ b/tests/models/opt/test_tokenization_opt.py @@ -15,7 +15,7 @@ import os import unittest -from transformers import OPTTokenizer, OPTTokenizerFast, BatchEncoding +from transformers import BatchEncoding, OPTTokenizer, OPTTokenizerFast from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES from transformers.testing_utils import require_tokenizers, require_torch from transformers.utils import cached_property From 1e21983a4ed08ca2c91d2d218e5048ba50998efa Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Wed, 4 May 2022 19:25:15 +0200 Subject: [PATCH 006/145] few changes - remove a test file - added the logits test --- tests/models/opt/test_modeling_opt.py | 34 +++- tests/models/opt/test_tokenization_opt.py | 185 ---------------------- 2 files changed, 33 insertions(+), 186 deletions(-) delete mode 100644 tests/models/opt/test_tokenization_opt.py diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index c0c92fcd341ef..46d10db74e904 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -21,7 +21,7 @@ import timeout_decorator # noqa -from transformers import OPTConfig, is_torch_available +from transformers import OPTConfig, GPT2Tokenizer, is_torch_available from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device from transformers.utils import cached_property @@ -966,3 +966,35 @@ def test_decoder_model_attn_mask_past(self): def test_retain_grad_hidden_states_attentions(self): # decoder cannot keep gradients return + +class OPTEmbeddingsTest(unittest.TestCase): + def setUp(self): + super().setUp() + self.path_model = "/home/younes/Desktop/Work/data/opt-350-m/" # TODO add the model on the hub + self.path_logits_meta = "/home/younes/Desktop/Work/metaseq-conversion/logits_metaseq_gpt2_tokenizer.p" # TODO add the logits somewhere? + + @unittest.skip("Skipping unless we find a solution to store the model") + def test_load_model(self): + try: + _ = OPTForCausalLM.from_pretrained(self.path_model) + except BaseException: + self.fail("Failed loading model") + + @unittest.skip("Skipping unless we find a solution to store the logits") + def test_logits(self): + model = OPTForCausalLM.from_pretrained(self.path_model) + model = model.eval() + # tokenizer = GPT2Tokenizer.from_pretrained("patrickvonplaten/opt_gpt2_tokenizer") + tokenizer = GPT2Tokenizer.from_pretrained("patrickvonplaten/opt_gpt2_tokenizer") + tokenizer.add_special_tokens({'pad_token': ''}) + + prompts = [ + "Today is a beautiful day and I want to", + "In the city of", + "Paris is the capital of France and", + "Computers and mobile phones have taken", + ] + input_ids = tokenizer(prompts, return_tensors="pt", padding=True).input_ids + logits = model(input_ids)[0] + logits_meta = torch.load(self.path_logits_meta) + assert torch.allclose(logits, logits_meta) \ No newline at end of file diff --git a/tests/models/opt/test_tokenization_opt.py b/tests/models/opt/test_tokenization_opt.py deleted file mode 100644 index 7dc93eb9ca23b..0000000000000 --- a/tests/models/opt/test_tokenization_opt.py +++ /dev/null @@ -1,185 +0,0 @@ -# Copyright 2020 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import json -import os -import unittest - -from transformers import BatchEncoding, OPTTokenizer, OPTTokenizerFast -from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES -from transformers.testing_utils import require_tokenizers, require_torch -from transformers.utils import cached_property - -from ...test_tokenization_common import TokenizerTesterMixin, filter_roberta_detectors - - -@require_tokenizers -class TestTokenizationOPT(TokenizerTesterMixin, unittest.TestCase): - tokenizer_class = OPTTokenizer - rust_tokenizer_class = OPTTokenizerFast - test_rust_tokenizer = True - from_pretrained_filter = filter_roberta_detectors - # from_pretrained_kwargs = {'add_prefix_space': True} - - def setUp(self): - super().setUp() - vocab = [ - "l", - "o", - "w", - "e", - "r", - "s", - "t", - "i", - "d", - "n", - "\u0120", - "\u0120l", - "\u0120n", - "\u0120lo", - "\u0120low", - "er", - "\u0120lowest", - "\u0120newer", - "\u0120wider", - "", - ] - vocab_tokens = dict(zip(vocab, range(len(vocab)))) - merges = ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""] - self.special_tokens_map = {"unk_token": ""} - - self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"]) - self.merges_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["merges_file"]) - with open(self.vocab_file, "w", encoding="utf-8") as fp: - fp.write(json.dumps(vocab_tokens) + "\n") - with open(self.merges_file, "w", encoding="utf-8") as fp: - fp.write("\n".join(merges)) - - def get_tokenizer(self, **kwargs): - kwargs.update(self.special_tokens_map) - return self.tokenizer_class.from_pretrained(self.tmpdirname, **kwargs) - - def get_rust_tokenizer(self, **kwargs): - kwargs.update(self.special_tokens_map) - return self.rust_tokenizer_class.from_pretrained(self.tmpdirname, **kwargs) - - def get_input_output_texts(self, tokenizer): - return "lower newer", "lower newer" - - @cached_property - def default_tokenizer(self): - return OPTTokenizer.from_pretrained("facebook/opt-large") - - @cached_property - def default_tokenizer_fast(self): - return OPTTokenizerFast.from_pretrained("facebook/opt-large") - - @require_torch - def test_prepare_batch(self): - src_text = ["A long paragraph for summarization.", "Another paragraph for summarization."] - expected_src_tokens = [0, 250, 251, 17818, 13, 39186, 1938, 4, 2] - - for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]: - batch = tokenizer(src_text, max_length=len(expected_src_tokens), padding=True, return_tensors="pt") - self.assertIsInstance(batch, BatchEncoding) - - self.assertEqual((2, 9), batch.input_ids.shape) - self.assertEqual((2, 9), batch.attention_mask.shape) - result = batch.input_ids.tolist()[0] - self.assertListEqual(expected_src_tokens, result) - # Test that special tokens are reset - - @require_torch - def test_prepare_batch_empty_target_text(self): - src_text = ["A long paragraph for summarization.", "Another paragraph for summarization."] - for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]: - batch = tokenizer(src_text, padding=True, return_tensors="pt") - # check if input_ids are returned and no labels - self.assertIn("input_ids", batch) - self.assertIn("attention_mask", batch) - self.assertNotIn("labels", batch) - self.assertNotIn("decoder_attention_mask", batch) - - @require_torch - def test_as_target_tokenizer_target_length(self): - tgt_text = [ - "Summary of the text.", - "Another summary.", - ] - for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]: - with tokenizer.as_target_tokenizer(): - targets = tokenizer(tgt_text, max_length=32, padding="max_length", return_tensors="pt") - self.assertEqual(32, targets["input_ids"].shape[1]) - - @require_torch - def test_prepare_batch_not_longer_than_maxlen(self): - for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]: - batch = tokenizer( - ["I am a small frog" * 1024, "I am a small frog"], padding=True, truncation=True, return_tensors="pt" - ) - self.assertIsInstance(batch, BatchEncoding) - self.assertEqual(batch.input_ids.shape, (2, 1024)) - - @require_torch - def test_special_tokens(self): - - src_text = ["A long paragraph for summarization."] - tgt_text = [ - "Summary of the text.", - ] - for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]: - inputs = tokenizer(src_text, return_tensors="pt") - with tokenizer.as_target_tokenizer(): - targets = tokenizer(tgt_text, return_tensors="pt") - input_ids = inputs["input_ids"] - labels = targets["input_ids"] - self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()) - self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()) - self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()) - self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()) - - def test_pretokenized_inputs(self): - pass - - def test_embeded_special_tokens(self): - for tokenizer, pretrained_name, kwargs in self.tokenizers_list: - with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"): - tokenizer_r = self.rust_tokenizer_class.from_pretrained(pretrained_name, **kwargs) - tokenizer_p = self.tokenizer_class.from_pretrained(pretrained_name, **kwargs) - sentence = "A, AllenNLP sentence." - tokens_r = tokenizer_r.encode_plus(sentence, add_special_tokens=True, return_token_type_ids=True) - tokens_p = tokenizer_p.encode_plus(sentence, add_special_tokens=True, return_token_type_ids=True) - - # token_type_ids should put 0 everywhere - self.assertEqual(sum(tokens_r["token_type_ids"]), sum(tokens_p["token_type_ids"])) - - # attention_mask should put 1 everywhere, so sum over length should be 1 - self.assertEqual( - sum(tokens_r["attention_mask"]) / len(tokens_r["attention_mask"]), - sum(tokens_p["attention_mask"]) / len(tokens_p["attention_mask"]), - ) - - tokens_r_str = tokenizer_r.convert_ids_to_tokens(tokens_r["input_ids"]) - tokens_p_str = tokenizer_p.convert_ids_to_tokens(tokens_p["input_ids"]) - - # Rust correctly handles the space before the mask while python doesnt - self.assertSequenceEqual(tokens_p["input_ids"], [0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]) - self.assertSequenceEqual(tokens_r["input_ids"], [0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]) - - self.assertSequenceEqual( - tokens_p_str, ["", "A", ",", "", "ĠAllen", "N", "LP", "Ġsentence", ".", ""] - ) - self.assertSequenceEqual( - tokens_r_str, ["", "A", ",", "", "ĠAllen", "N", "LP", "Ġsentence", ".", ""] - ) From 13632212d5bacc83bb9a9b8b6816a70bde865dbc Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Wed, 4 May 2022 19:26:06 +0200 Subject: [PATCH 007/145] Update src/transformers/models/auto/tokenization_auto.py Co-authored-by: Patrick von Platen --- src/transformers/models/auto/tokenization_auto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 7d688ba13ff70..df80135553ca7 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -107,7 +107,7 @@ ("blenderbot", ("BlenderbotTokenizer", "BlenderbotTokenizerFast")), ("tapex", ("TapexTokenizer", None)), ("bart", ("BartTokenizer", "BartTokenizerFast")), - ("opt", ("OPTTokenizer", "OPTTokenizerFast")), + ("opt", ("GPT2Tokenizer", "GPT2TokenizerFast")), ("longformer", ("LongformerTokenizer", "LongformerTokenizerFast" if is_tokenizers_available() else None)), ("roberta", ("RobertaTokenizer", "RobertaTokenizerFast" if is_tokenizers_available() else None)), ( From 842727999cb6d7a298963021dcd5daafb98d2453 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Wed, 4 May 2022 19:54:56 +0200 Subject: [PATCH 008/145] add gen tests --- tests/models/opt/test_modeling_opt.py | 41 +++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index 46d10db74e904..3344fc6f37134 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -21,7 +21,7 @@ import timeout_decorator # noqa -from transformers import OPTConfig, GPT2Tokenizer, is_torch_available +from transformers import OPTConfig, GPT2Tokenizer, is_torch_available, pipeline from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device from transformers.utils import cached_property @@ -997,4 +997,41 @@ def test_logits(self): input_ids = tokenizer(prompts, return_tensors="pt", padding=True).input_ids logits = model(input_ids)[0] logits_meta = torch.load(self.path_logits_meta) - assert torch.allclose(logits, logits_meta) \ No newline at end of file + assert torch.allclose(logits, logits_meta) + +class OPTGenerationTest(unittest.TestCase): + def setUp(self): + super().setUp() + self.path_model = "/home/younes/Desktop/Work/data/opt-350-m/" # TODO add the model on the hub + self.path_logits_meta = "/home/younes/Desktop/Work/metaseq-conversion/logits_metaseq_gpt2_tokenizer.p" # TODO add the logits somewhere? + + + @unittest.skip("Skipping unless we find a solution to store the model") + def test_generation(self): + model = OPTForCausalLM.from_pretrained(self.path_model) + model = model.eval() + # tokenizer = GPT2Tokenizer.from_pretrained("patrickvonplaten/opt_gpt2_tokenizer") + tokenizer = GPT2Tokenizer.from_pretrained("patrickvonplaten/opt_gpt2_tokenizer") + #tokenizer.add_special_tokens({'pad_token': ''}) + model.config.eos_token_id = tokenizer.eos_token_id + + gen = pipeline("text-generation", model=model, tokenizer=tokenizer, return_tensors=True) + + prompts = [ + "Today is a beautiful day and I want to", + "In the city of", + "Paris is the capital of France and", + "Computers and mobile phones have taken", + ] + NEXT_TOKENS = [ + 3392, + 764, + 5, + 81, + ] + GEN_OUTPUT = [] + for prompt in prompts: + len_input_sentence = len(tokenizer.tokenize(prompt)) + predicted_next_token = gen(prompt)[0]['generated_token_ids'][len_input_sentence] + GEN_OUTPUT.append(predicted_next_token) + self.assertListEqual(GEN_OUTPUT, NEXT_TOKENS) \ No newline at end of file From 5e8e2f599be59d9a3f9692c93318b6a091005a46 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Wed, 4 May 2022 23:50:12 +0200 Subject: [PATCH 009/145] few changes - rm mask filling example on docstring --- src/transformers/models/opt/modeling_opt.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 01f5bdfba0080..9bf0cb595dfa8 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -545,26 +545,6 @@ def __init_subclass__(self): >>> tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] 'PG&E scheduled the blackouts in response to forecasts for high winds amid dry conditions' ``` - - Mask filling example: - - ```python - >>> from transformers import OPTTokenizer, OPTForConditionalGeneration - - >>> tokenizer = OPTTokenizer.from_pretrained("") - >>> model = OPTForConditionalGeneration.from_pretrained("") - - >>> TXT = "My friends are but they eat too many carbs." - >>> input_ids = tokenizer([TXT], return_tensors="pt")["input_ids"] - >>> logits = model(input_ids).logits - - >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero().item() - >>> probs = logits[0, masked_index].softmax(dim=0) - >>> values, predictions = probs.topk(5) - - >>> tokenizer.decode(predictions).split() - ['not', 'good', 'healthy', 'great', 'very'] - ``` """ OPT_INPUTS_DOCSTRING = r""" From be0e434a55015638d5de9821748d4cbf08e54d80 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Wed, 4 May 2022 23:57:37 +0200 Subject: [PATCH 010/145] few changes - remove useless args --- src/transformers/models/opt/modeling_opt.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 9bf0cb595dfa8..cc2c1356c126b 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -232,11 +232,7 @@ def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): def forward( self, hidden_states: torch.Tensor, - key_value_states: Optional[torch.Tensor] = None, - past_key_value: Optional[Tuple[torch.Tensor]] = None, attention_mask: Optional[torch.Tensor] = None, - layer_head_mask: Optional[torch.Tensor] = None, - output_attentions: bool = False, self_attn_padding_mask: Optional[Tensor] = None, ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: """Input shape: Batch x Time x Channel""" @@ -400,14 +396,11 @@ def forward( # Self Attention # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 - self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None + # self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None # add present self-attn cache to positions 1,2 of present_key_value tuple hidden_states, self_attn_weights = self.self_attn( hidden_states=hidden_states, - past_key_value=self_attn_past_key_value, attention_mask=attention_mask, - layer_head_mask=layer_head_mask, - output_attentions=output_attentions, self_attn_padding_mask=self_attn_padding_mask, ) hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) From 51db79e13f284ea0c518acdd601d84bfb2c2d661 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Thu, 5 May 2022 15:30:53 +0200 Subject: [PATCH 011/145] some changes - more tests should pass now - needs to clean more - documentation still needs to be done --- docs/source/en/index.mdx | 1 + docs/source/en/model_doc/opt.mdx | 15 + docs/source/en/serialization.mdx | 1 + .../models/opt/configuration_opt.py | 65 +- src/transformers/models/opt/modeling_opt.py | 580 ++---------------- .../models/opt/modeling_tf_opt.py | 77 +-- src/transformers/utils/dummy_flax_objects.py | 49 ++ src/transformers/utils/dummy_pt_objects.py | 52 ++ src/transformers/utils/dummy_tf_objects.py | 21 + .../utils/dummy_tokenizers_objects.py | 7 + tests/models/opt/test_embeddings_opt.py | 57 -- tests/models/opt/test_modeling_opt.py | 28 +- utils/check_repo.py | 2 + 13 files changed, 270 insertions(+), 685 deletions(-) delete mode 100644 tests/models/opt/test_embeddings_opt.py diff --git a/docs/source/en/index.mdx b/docs/source/en/index.mdx index 1a5829db217a9..6550f434da69c 100644 --- a/docs/source/en/index.mdx +++ b/docs/source/en/index.mdx @@ -229,6 +229,7 @@ Flax), PyTorch, and/or TensorFlow. | Nystromformer | ❌ | ❌ | ✅ | ❌ | ❌ | | OpenAI GPT | ✅ | ✅ | ✅ | ✅ | ❌ | | OpenAI GPT-2 | ✅ | ✅ | ✅ | ✅ | ✅ | +| OPT | ✅ | ✅ | ✅ | ✅ | ✅ | | Pegasus | ✅ | ✅ | ✅ | ✅ | ✅ | | Perceiver | ✅ | ❌ | ✅ | ❌ | ❌ | | PLBart | ✅ | ❌ | ✅ | ❌ | ❌ | diff --git a/docs/source/en/model_doc/opt.mdx b/docs/source/en/model_doc/opt.mdx index 742261ceb02f3..e1a37320cd719 100644 --- a/docs/source/en/model_doc/opt.mdx +++ b/docs/source/en/model_doc/opt.mdx @@ -49,6 +49,16 @@ The original code can be found [here](). [[autodoc]] OPTModel - forward +## OPTPretrainedModel + +[[autodoc]] OPTPretrainedModel + - all + +## PretrainedOPTModel + +[[autodoc]] PretrainedOPTModel + - all + ## OPTForConditionalGeneration [[autodoc]] OPTForConditionalGeneration @@ -74,6 +84,11 @@ The original code can be found [here](). [[autodoc]] TFOPTModel - call +## TFOPTPretrainedModel + +[[autodoc]] TFOPTPretrainedModel + - call + ## TFOPTForConditionalGeneration [[autodoc]] TFOPTForConditionalGeneration diff --git a/docs/source/en/serialization.mdx b/docs/source/en/serialization.mdx index 4ae5c9a57ecbd..4391c6024ab97 100644 --- a/docs/source/en/serialization.mdx +++ b/docs/source/en/serialization.mdx @@ -68,6 +68,7 @@ Ready-made configurations include the following architectures: - Marian - mBART - OpenAI GPT-2 +- OPT - PLBart - RoBERTa - RoFormer diff --git a/src/transformers/models/opt/configuration_opt.py b/src/transformers/models/opt/configuration_opt.py index 1629a596c9cc5..7d341439006c1 100644 --- a/src/transformers/models/opt/configuration_opt.py +++ b/src/transformers/models/opt/configuration_opt.py @@ -48,18 +48,12 @@ class OPTConfig(PretrainedConfig): `inputs_ids` passed when calling [`OPTModel`] or [`TFOPTModel`]. d_model (`int`, *optional*, defaults to 1024): Dimensionality of the layers and the pooler layer. - encoder_layers (`int`, *optional*, defaults to 12): - Number of encoder layers. - decoder_layers (`int`, *optional*, defaults to 12): + num_layers (`int`, *optional*, defaults to 12): Number of decoder layers. - encoder_attention_heads (`int`, *optional*, defaults to 16): - Number of attention heads for each attention layer in the Transformer encoder. - decoder_attention_heads (`int`, *optional*, defaults to 16): - Number of attention heads for each attention layer in the Transformer decoder. - decoder_ffn_dim (`int`, *optional*, defaults to 4096): - Dimensionality of the "intermediate" (often named feed-forward) layer in decoder. - encoder_ffn_dim (`int`, *optional*, defaults to 4096): + ffn_dim (`int`, *optional*, defaults to 4096): Dimensionality of the "intermediate" (often named feed-forward) layer in decoder. + num_attention_heads (`int`, *optional*, defaults to 16): + Number of attention heads for each attention layer in the Transformer decoder. activation_function (`str` or `function`, *optional*, defaults to `"gelu"`): The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported. @@ -69,25 +63,18 @@ class OPTConfig(PretrainedConfig): The dropout ratio for the attention probabilities. activation_dropout (`float`, *optional*, defaults to 0.0): The dropout ratio for activations inside the fully connected layer. - classifier_dropout (`float`, *optional*, defaults to 0.0): - The dropout ratio for classifier. max_position_embeddings (`int`, *optional*, defaults to 1024): The maximum sequence length that this model might ever be used with. Typically set this to something large just in case (e.g., 512 or 1024 or 2048). init_std (`float`, *optional*, defaults to 0.02): The standard deviation of the truncated_normal_initializer for initializing all weight matrices. - encoder_layerdrop: (`float`, *optional*, defaults to 0.0): - The LayerDrop probability for the encoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556) - for more details. - decoder_layerdrop: (`float`, *optional*, defaults to 0.0): - The LayerDrop probability for the decoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556) - for more details. + layerdrop: (`float`, *optional*, defaults to 0.0): + The LayerDrop probability. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556) for more + details. scale_embedding (`bool`, *optional*, defaults to `False`): Scale embeddings by diving by sqrt(d_model). use_cache (`bool`, *optional*, defaults to `True`): Whether or not the model should return the last key/values attentions (not used by all models). - num_labels: (`int`, *optional*, defaults to 3): - The number of labels to use in [`OPTForSequenceClassification`]. forced_eos_token_id (`int`, *optional*, defaults to 2): The id of the token to force as the last generated token when `max_length` is reached. Usually set to `eos_token_id`. @@ -108,16 +95,16 @@ class OPTConfig(PretrainedConfig): ```""" model_type = "opt" keys_to_ignore_at_inference = ["past_key_values"] - attribute_map = {"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"} + attribute_map = {"hidden_size": "d_model"} def __init__( self, vocab_size=50272, # TODO check the real value max_position_embeddings=2048, - decoder_layers=24, - decoder_ffn_dim=4096, - decoder_attention_heads=16, - decoder_layerdrop=0.0, + num_layers=24, + num_attention_heads=16, + ffn_dim=4096, + layerdrop=0.0, activation_function="gelu", d_model=1024, embed_dim=512, @@ -125,43 +112,37 @@ def __init__( attention_dropout=0.0, activation_dropout=0.0, init_std=0.02, - classifier_dropout=0.0, scale_embedding=False, share_input_output_embed=True, use_cache=False, - num_labels=3, pad_token_id=1, bos_token_id=0, eos_token_id=2, - is_encoder_decoder=True, decoder_start_token_id=2, forced_eos_token_id=2, **kwargs ): self.vocab_size = vocab_size self.max_position_embeddings = max_position_embeddings + self.num_attention_heads = num_attention_heads self.embed_dim = embed_dim + self.ffn_dim = ffn_dim self.share_input_output_embed = share_input_output_embed self.d_model = d_model - self.decoder_ffn_dim = decoder_ffn_dim - self.decoder_layers = decoder_layers - self.decoder_attention_heads = decoder_attention_heads + self.num_layers = num_layers self.dropout = dropout self.attention_dropout = attention_dropout self.activation_dropout = activation_dropout self.activation_function = activation_function self.init_std = init_std - self.decoder_layerdrop = decoder_layerdrop - self.classifier_dropout = classifier_dropout + self.layerdrop = layerdrop self.use_cache = use_cache self.scale_embedding = scale_embedding # scale factor will be sqrt(d_model) if True super().__init__( - num_labels=num_labels, pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, - is_encoder_decoder=is_encoder_decoder, decoder_start_token_id=decoder_start_token_id, forced_eos_token_id=forced_eos_token_id, **kwargs, @@ -261,7 +242,7 @@ def _generate_dummy_inputs_for_default_and_seq2seq_lm( import torch batch, encoder_seq_length = common_inputs["input_ids"].shape decoder_seq_length = common_inputs["decoder_input_ids"].shape[1] - num_encoder_attention_heads, num_decoder_attention_heads = self.num_attention_heads + num_encoder_attention_heads, num_num_attention_heads = self.num_attention_heads encoder_shape = ( batch, num_encoder_attention_heads, @@ -271,9 +252,9 @@ def _generate_dummy_inputs_for_default_and_seq2seq_lm( decoder_past_length = decoder_seq_length + 3 decoder_shape = ( batch, - num_decoder_attention_heads, + num_num_attention_heads, decoder_past_length, - self._config.hidden_size // num_decoder_attention_heads, + self._config.hidden_size // num_num_attention_heads, ) common_inputs["decoder_attention_mask"] = torch.cat( @@ -282,10 +263,10 @@ def _generate_dummy_inputs_for_default_and_seq2seq_lm( common_inputs["past_key_values"] = [] # If the number of encoder and decoder layers are present in the model configuration, both are considered - num_encoder_layers, num_decoder_layers = self.num_layers - min_num_layers = min(num_encoder_layers, num_decoder_layers) - max_num_layers = max(num_encoder_layers, num_decoder_layers) - min_num_layers - remaining_side_name = "encoder" if num_encoder_layers > num_decoder_layers else "decoder" + num_encoder_layers, num_layers = self.num_layers + min_num_layers = min(num_encoder_layers, num_layers) + max_num_layers = max(num_encoder_layers, num_layers) - min_num_layers + remaining_side_name = "encoder" if num_encoder_layers > num_layers else "decoder" for _ in range(min_num_layers): common_inputs["past_key_values"].append( diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index cc2c1356c126b..fd0f4491c74b1 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -118,26 +118,6 @@ def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] return inverted_mask.masked_fill(inverted_mask.bool(), torch.finfo(dtype).min) -# Copied from transformers.models.bart.modeling_bart.BartLearnedPositionalEmbedding with Bart->OPT -# class OPTLearnedPositionalEmbedding(nn.Embedding): -# """ -# This module learns positional embeddings up to a fixed maximum size. #""" - -# def __init__(self, num_embeddings: int, embedding_dim: int): -# # OPT is set up so that if padding_idx is specified then offset the embedding ids by 2 -# # and adjust num_embeddings appropriately. Other models don't have this hack -# self.offset = 2 -# super().__init__(num_embeddings + self.offset, embedding_dim) - -# def forward(self, input_ids_shape: torch.Size, past_key_values_length: int = 0): -# """`input_ids_shape` is expected to be [bsz x seqlen].""" -# bsz, seq_len = input_ids_shape[:2] -# positions = torch.arange( -# past_key_values_length, past_key_values_length + seq_len, dtype=torch.long, device=self.weight.device -# ) -# return super().forward(positions + self.offset) - - def make_positions(tensor, padding_idx: int, onnx_trace: bool = False): """Replace non-padding symbols with their position numbers. @@ -151,7 +131,7 @@ def make_positions(tensor, padding_idx: int, onnx_trace: bool = False): return (torch.cumsum(mask, dim=1).type_as(mask) * mask).long() + padding_idx -class LearnedPositionalEmbedding(nn.Embedding): +class OPTLearnedPositionalEmbedding(nn.Embedding): """ This module learns positional embeddings up to a fixed maximum size. Padding ids are ignored by either offsetting based on padding_idx or by setting padding_idx to None and ensuring that the appropriate position ids are passed to @@ -173,9 +153,10 @@ def forward( positions: Optional[Tensor] = None, ): """Input is expected to be of size [bsz x seqlen].""" - assert (positions is None) or ( + if not ((positions is None) or ( self.padding_idx is None - ), "If positions is pre-computed then padding_idx should not be set." + )): + raise ValueError("If positions is pre-computed then padding_idx should not be set.") # we cannot use incremental state here because we must be aware of # padding. @@ -193,7 +174,6 @@ def forward( ) -# Copied from transformers.models.bart.modeling_bart.BartAttention with Bart->OPT class OPTAttention(nn.Module): """Multi-headed attention from 'Attention Is All You Need' paper""" @@ -237,12 +217,6 @@ def forward( ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: """Input shape: Batch x Time x Channel""" - # if key_value_states are provided this layer is used as a cross-attention layer - # for the decoder - - # create key padding mask - # HACK : to fix - assert torch.equal(attention_mask[0, 0, :, :], attention_mask[-1, 0, :, :]) attention_mask = attention_mask[0, 0, :, :] return F.multi_head_attention_forward( @@ -270,7 +244,6 @@ def forward( ) -# Copied from transformers.models.bart.modeling_bart.BartEncoderLayer with Bart->OPT class OPTEncoderLayer(nn.Module): def __init__(self, config: OPTConfig): super().__init__() @@ -339,7 +312,6 @@ def forward( return outputs -# Copied from transformers.models.bart.modeling_bart.BartDecoderLayer with Bart->OPT class OPTDecoderLayer(nn.Module): def __init__(self, config: OPTConfig): super().__init__() @@ -347,7 +319,7 @@ def __init__(self, config: OPTConfig): self.self_attn = OPTAttention( embed_dim=self.embed_dim, - num_heads=config.decoder_attention_heads, + num_heads=config.num_attention_heads, dropout=config.attention_dropout, is_decoder=True, ) @@ -358,8 +330,8 @@ def __init__(self, config: OPTConfig): self.activation_dropout = config.activation_dropout self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim) - self.fc1 = nn.Linear(self.embed_dim, config.decoder_ffn_dim) - self.fc2 = nn.Linear(config.decoder_ffn_dim, self.embed_dim) + self.fc1 = nn.Linear(self.embed_dim, config.ffn_dim) + self.fc2 = nn.Linear(config.ffn_dim, self.embed_dim) self.final_layer_norm = nn.LayerNorm(self.embed_dim) def forward( @@ -384,14 +356,13 @@ def forward( layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size `(encoder_attention_heads,)`. cross_attn_layer_head_mask (`torch.FloatTensor`): mask for cross-attention heads in a given layer of - size `(decoder_attention_heads,)`. + size `(num_attention_heads,)`. past_key_value (`Tuple(torch.FloatTensor)`): cached past key and value projection states output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. """ - # hidden_states = hidden_states.permute(1, 0, 2) residual = hidden_states # Self Attention @@ -434,7 +405,6 @@ def forward( return outputs -# Copied from transformers.models.bart.modeling_bart.BartClassificationHead with Bart->OPT class OPTClassificationHead(nn.Module): """Head for sentence-level classification tasks.""" @@ -458,8 +428,26 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: hidden_states = self.out_proj(hidden_states) return hidden_states +OPT_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`OPTConfig`]): + Model configuration class with all the parameters of the model. Initializing with a config file does not + load the weights associated with the model, only the configuration. Check out the + [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" -# Copied from transformers.models.bart.modeling_bart.BartPretrainedModel with Bart->OPT +@add_start_docstrings( + "The bare OPT Model outputting raw hidden-states without any specific head on top.", + OPT_START_DOCSTRING, +) class OPTPretrainedModel(PreTrainedModel): config_class = OPTConfig base_model_prefix = "model" @@ -476,12 +464,12 @@ def _init_weights(self, module): module.weight.data.normal_(mean=0.0, std=std) if module.padding_idx is not None: module.weight.data[module.padding_idx].zero_() - def _set_gradient_checkpointing(self, module, value=False): if isinstance(module, (OPTDecoder)): module.gradient_checkpointing = value @property + @add_start_docstrings_to_model_forward(OPT_START_DOCSTRING) def dummy_inputs(self): pad_token = self.config.pad_token_id input_ids = torch.tensor([[0, 6, 10, 4, 2], [0, 8, 12, 2, pad_token]], device=self.device) @@ -491,52 +479,22 @@ def dummy_inputs(self): } return dummy_inputs - -# Copied from transformers.models.bart.modeling_bart.PretrainedBartModel with Bart->OPT -class PretrainedOPTModel(OPTPretrainedModel): - def __init_subclass__(self): - warnings.warn( - "The class `PretrainedOPTModel` has been depreciated, please use `OPTPretrainedModel` instead.", - FutureWarning, - ) - - -OPT_START_DOCSTRING = r""" - This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. - Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage - and behavior. - - Parameters: - config ([`OPTConfig`]): - Model configuration class with all the parameters of the model. Initializing with a config file does not - load the weights associated with the model, only the configuration. Check out the - [`~PreTrainedModel.from_pretrained`] method to load the model weights. -""" - OPT_GENERATION_EXAMPLE = r""" - Summarization example: + Generation example: ```python - >>> from transformers import OPTTokenizer, OPTForConditionalGeneration + >>> from transformers import GPT2Tokenizer, OPTForCausalLM - >>> model = OPTForConditionalGeneration.from_pretrained("facebook/opt-large-cnn") - >>> tokenizer = OPTTokenizer.from_pretrained("facebook/opt-large-cnn") + >>> model = OPTForCausalLM.from_pretrained("facebook/opt-large-cnn") + >>> tokenizer = GPT2Tokenizer.from_pretrained("facebook/opt-large-cnn") - >>> ARTICLE_TO_SUMMARIZE = ( - ... "PG&E stated it scheduled the blackouts in response to forecasts for high winds " - ... "amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were " - ... "scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow." - ... ) - >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors="pt") + >>> TEXTS_TO_GENERATE = "Hey, are you consciours? Can you talk to me?" "Hi there, my name is Barack" + >>> inputs = tokenizer([TEXTS_TO_GENERATE], max_length=1024, return_tensors="pt") >>> # Generate Summary >>> summary_ids = model.generate(inputs["input_ids"], num_beams=2, min_length=0, max_length=20) >>> tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] - 'PG&E scheduled the blackouts in response to forecasts for high winds amid dry conditions' + 'I'm not conscious.<\s>' ``` """ @@ -557,24 +515,13 @@ def __init_subclass__(self): - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. - [What are decoder input IDs?](../glossary#decoder-input-ids) - OPT uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). - For translation and summarization training, `decoder_input_ids` should be provided. If no - `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right - for denoising pre-training following the paper. - decoder_attention_mask (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_inputs`] and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. @@ -584,19 +531,6 @@ def __init_subclass__(self): - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. - decoder_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules in the decoder. Mask values selected in `[0, - 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of @@ -615,14 +549,6 @@ def __init_subclass__(self): `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - decoder_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded - representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be - input (see `past_key_values`). This is useful if you want more control over how to convert - `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. - - If `decoder_input_ids` and `decoder_inputs_embeds` are both unset, `decoder_inputs_embeds` takes the value - of `inputs_embeds`. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). @@ -637,10 +563,9 @@ def __init_subclass__(self): """ -# Copied from transformers.models.bart.modeling_bart.BartDecoder with Bart->OPT class OPTDecoder(OPTPretrainedModel): """ - Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`OPTDecoderLayer`] + Transformer decoder consisting of *config.num_layers* layers. Each layer is a [`OPTDecoderLayer`] Args: config: OPTConfig @@ -650,7 +575,7 @@ class OPTDecoder(OPTPretrainedModel): def __init__(self, config: OPTConfig): super().__init__(config) self.dropout = config.dropout - self.layerdrop = config.decoder_layerdrop + self.layerdrop = config.layerdrop self.padding_idx = config.pad_token_id self.max_target_positions = config.max_position_embeddings self.share_input_output_embed = config.share_input_output_embed @@ -662,7 +587,7 @@ def __init__(self, config: OPTConfig): if self.padding_idx is not None: num_embeddings = config.max_position_embeddings + self.padding_idx + 1 - self.embed_positions = LearnedPositionalEmbedding( + self.embed_positions = OPTLearnedPositionalEmbedding( num_embeddings, config.d_model, self.padding_idx, @@ -678,7 +603,7 @@ def __init__(self, config: OPTConfig): self.output_projection = nn.Linear(config.embed_dim, config.vocab_size, bias=False) - self.layers = nn.ModuleList([OPTDecoderLayer(config) for _ in range(config.decoder_layers)]) + self.layers = nn.ModuleList([OPTDecoderLayer(config) for _ in range(config.num_layers)]) self.gradient_checkpointing = False # Initialize weights and apply final processing @@ -752,13 +677,13 @@ def forward( - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) - head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + head_mask (`torch.Tensor` of shape `(num_layers, num_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + cross_attn_head_mask (`torch.Tensor` of shape `(num_layers, num_attention_heads)`, *optional*): Mask to nullify selected heads of the cross-attention modules in the decoder to avoid performing cross-attention on hidden heads. Mask values selected in `[0, 1]`: @@ -936,7 +861,6 @@ def custom_forward(*inputs): "The bare OPT Model outputting raw hidden-states without any specific head on top.", OPT_START_DOCSTRING, ) -# Copied from transformers.models.bart.modeling_bart.BartModel with BART->OPT,Bart->OPT class OPTModel(OPTPretrainedModel): def __init__(self, config: OPTConfig): super().__init__(config) @@ -1051,423 +975,6 @@ def forward( ) -@add_start_docstrings( - "The OPT Model with a language modeling head. Can be used for summarization.", OPT_START_DOCSTRING -) -# Copied from transformers.models.bart.modeling_bart.BartForConditionalGeneration with BART->OPT,Bart->OPT -class OPTForConditionalGeneration(OPTPretrainedModel): - base_model_prefix = "model" - _keys_to_ignore_on_load_missing = [r"final_logits_bias", r"lm_head\.weight"] - - def __init__(self, config: OPTConfig): - super().__init__(config) - self.model = OPTModel(config) - self.register_buffer("final_logits_bias", torch.zeros((1, self.model.decoder.vocab_size))) - self.lm_head = nn.Linear(config.d_model, self.model.decoder.vocab_size, bias=False) - - # Initialize weights and apply final processing - self.post_init() - - def get_encoder(self): - return self.model.get_encoder() - - def get_decoder(self): - return self.model.get_decoder() - - def resize_token_embeddings(self, new_num_tokens: int) -> nn.Embedding: - new_embeddings = super().resize_token_embeddings(new_num_tokens) - self._resize_final_logits_bias(new_num_tokens) - return new_embeddings - - def _resize_final_logits_bias(self, new_num_tokens: int) -> None: - old_num_tokens = self.final_logits_bias.shape[-1] - if new_num_tokens <= old_num_tokens: - new_bias = self.final_logits_bias[:, :new_num_tokens] - else: - extra_bias = torch.zeros((1, new_num_tokens - old_num_tokens), device=self.final_logits_bias.device) - new_bias = torch.cat([self.final_logits_bias, extra_bias], dim=1) - self.register_buffer("final_logits_bias", new_bias) - - def get_output_embeddings(self): - return self.lm_head - - def set_output_embeddings(self, new_embeddings): - self.lm_head = new_embeddings - - @add_start_docstrings_to_model_forward(OPT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=Seq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) - @add_end_docstrings(OPT_GENERATION_EXAMPLE) - def forward( - self, - input_ids: torch.LongTensor = None, - attention_mask: Optional[torch.Tensor] = None, - decoder_input_ids: Optional[torch.LongTensor] = None, - decoder_attention_mask: Optional[torch.LongTensor] = None, - head_mask: Optional[torch.Tensor] = None, - decoder_head_mask: Optional[torch.Tensor] = None, - cross_attn_head_mask: Optional[torch.Tensor] = None, - encoder_outputs: Optional[List[torch.FloatTensor]] = None, - past_key_values: Optional[List[torch.FloatTensor]] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - decoder_inputs_embeds: Optional[torch.FloatTensor] = None, - labels: Optional[torch.LongTensor] = None, - use_cache: Optional[bool] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - ) -> Union[Tuple, Seq2SeqLMOutput]: - r""" - labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - Returns: - """ - return_dict = return_dict if return_dict is not None else self.config.use_return_dict - - if labels is not None: - if use_cache: - logger.warning("The `use_cache` argument is changed to `False` since `labels` is provided.") - use_cache = False - if decoder_input_ids is None and decoder_inputs_embeds is None: - decoder_input_ids = shift_tokens_right( - labels, self.config.pad_token_id, self.config.decoder_start_token_id - ) - - outputs = self.model( - input_ids, - attention_mask=attention_mask, - decoder_input_ids=decoder_input_ids, - encoder_outputs=encoder_outputs, - decoder_attention_mask=decoder_attention_mask, - head_mask=head_mask, - decoder_head_mask=decoder_head_mask, - cross_attn_head_mask=cross_attn_head_mask, - past_key_values=past_key_values, - inputs_embeds=inputs_embeds, - decoder_inputs_embeds=decoder_inputs_embeds, - use_cache=use_cache, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - ) - lm_logits = self.lm_head(outputs[0]) + self.final_logits_bias - - masked_lm_loss = None - if labels is not None: - loss_fct = CrossEntropyLoss() - masked_lm_loss = loss_fct(lm_logits.view(-1, self.config.vocab_size), labels.view(-1)) - - if not return_dict: - output = (lm_logits,) + outputs[1:] - return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output - - return Seq2SeqLMOutput( - loss=masked_lm_loss, - logits=lm_logits, - past_key_values=outputs.past_key_values, - decoder_hidden_states=outputs.decoder_hidden_states, - decoder_attentions=outputs.decoder_attentions, - cross_attentions=outputs.cross_attentions, - encoder_last_hidden_state=outputs.encoder_last_hidden_state, - encoder_hidden_states=outputs.encoder_hidden_states, - encoder_attentions=outputs.encoder_attentions, - ) - - def prepare_inputs_for_generation( - self, - decoder_input_ids, - past=None, - attention_mask=None, - head_mask=None, - decoder_head_mask=None, - cross_attn_head_mask=None, - use_cache=None, - encoder_outputs=None, - **kwargs - ): - # cut decoder_input_ids if past is used - if past is not None: - decoder_input_ids = decoder_input_ids[:, -1:] - - return { - "input_ids": None, # encoder_outputs is defined. input_ids not needed - "encoder_outputs": encoder_outputs, - "past_key_values": past, - "decoder_input_ids": decoder_input_ids, - "attention_mask": attention_mask, - "head_mask": head_mask, - "decoder_head_mask": decoder_head_mask, - "cross_attn_head_mask": cross_attn_head_mask, - "use_cache": use_cache, # change this to avoid caching (presumably for debugging) - } - - def prepare_decoder_input_ids_from_labels(self, labels: torch.Tensor): - return shift_tokens_right(labels, self.config.pad_token_id, self.config.decoder_start_token_id) - - @staticmethod - def _reorder_cache(past, beam_idx): - reordered_past = () - for layer_past in past: - # cached cross_attention states don't have to be reordered -> they are always the same - reordered_past += ( - tuple(past_state.index_select(0, beam_idx) for past_state in layer_past[:2]) + layer_past[2:], - ) - return reordered_past - - -@add_start_docstrings( - """ - OPT model with a sequence classification/head on top (a linear layer on top of the pooled output) e.g. for GLUE - tasks. - """, - OPT_START_DOCSTRING, -) -# Copied from transformers.models.bart.modeling_bart.BartForSequenceClassification with BART->OPT,Bart->OPT -class OPTForSequenceClassification(OPTPretrainedModel): - def __init__(self, config: OPTConfig, **kwargs): - super().__init__(config, **kwargs) - self.model = OPTModel(config) - self.classification_head = OPTClassificationHead( - config.d_model, - config.d_model, - config.num_labels, - config.classifier_dropout, - ) - self.model._init_weights(self.classification_head.dense) - self.model._init_weights(self.classification_head.out_proj) - - @add_start_docstrings_to_model_forward(OPT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, - checkpoint=_CHECKPOINT_FOR_SEQUENCE_CLASSIFICATION, - output_type=Seq2SeqSequenceClassifierOutput, - config_class=_CONFIG_FOR_DOC, - expected_output=_SEQ_CLASS_EXPECTED_OUTPUT, - expected_loss=_SEQ_CLASS_EXPECTED_LOSS, - ) - def forward( - self, - input_ids: torch.LongTensor = None, - attention_mask: Optional[torch.Tensor] = None, - decoder_input_ids: Optional[torch.LongTensor] = None, - decoder_attention_mask: Optional[torch.LongTensor] = None, - head_mask: Optional[torch.Tensor] = None, - decoder_head_mask: Optional[torch.Tensor] = None, - cross_attn_head_mask: Optional[torch.Tensor] = None, - encoder_outputs: Optional[List[torch.FloatTensor]] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - decoder_inputs_embeds: Optional[torch.FloatTensor] = None, - labels: Optional[torch.LongTensor] = None, - use_cache: Optional[bool] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - ) -> Union[Tuple, Seq2SeqSequenceClassifierOutput]: - r""" - labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., - config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). - """ - return_dict = return_dict if return_dict is not None else self.config.use_return_dict - if labels is not None: - use_cache = False - - if input_ids is None and inputs_embeds is not None: - raise NotImplementedError( - f"Passing input embeddings is currently not supported for {self.__class__.__name__}" - ) - - outputs = self.model( - input_ids, - attention_mask=attention_mask, - decoder_input_ids=decoder_input_ids, - decoder_attention_mask=decoder_attention_mask, - head_mask=head_mask, - decoder_head_mask=decoder_head_mask, - cross_attn_head_mask=cross_attn_head_mask, - encoder_outputs=encoder_outputs, - inputs_embeds=inputs_embeds, - decoder_inputs_embeds=decoder_inputs_embeds, - use_cache=use_cache, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - ) - hidden_states = outputs[0] # last hidden state - - eos_mask = input_ids.eq(self.config.eos_token_id) - - if len(torch.unique_consecutive(eos_mask.sum(1))) > 1: - raise ValueError("All examples must have the same number of tokens.") - sentence_representation = hidden_states[eos_mask, :].view(hidden_states.size(0), -1, hidden_states.size(-1))[ - :, -1, : - ] - logits = self.classification_head(sentence_representation) - - loss = None - if labels is not None: - if self.config.problem_type is None: - if self.config.num_labels == 1: - self.config.problem_type = "regression" - elif self.config.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int): - self.config.problem_type = "single_label_classification" - else: - self.config.problem_type = "multi_label_classification" - - if self.config.problem_type == "regression": - loss_fct = MSELoss() - if self.config.num_labels == 1: - loss = loss_fct(logits.squeeze(), labels.squeeze()) - else: - loss = loss_fct(logits, labels) - elif self.config.problem_type == "single_label_classification": - loss_fct = CrossEntropyLoss() - loss = loss_fct(logits.view(-1, self.config.num_labels), labels.view(-1)) - elif self.config.problem_type == "multi_label_classification": - loss_fct = BCEWithLogitsLoss() - loss = loss_fct(logits, labels) - if not return_dict: - output = (logits,) + outputs[1:] - return ((loss,) + output) if loss is not None else output - - return Seq2SeqSequenceClassifierOutput( - loss=loss, - logits=logits, - past_key_values=outputs.past_key_values, - decoder_hidden_states=outputs.decoder_hidden_states, - decoder_attentions=outputs.decoder_attentions, - cross_attentions=outputs.cross_attentions, - encoder_last_hidden_state=outputs.encoder_last_hidden_state, - encoder_hidden_states=outputs.encoder_hidden_states, - encoder_attentions=outputs.encoder_attentions, - ) - - -@add_start_docstrings( - """ - OPT Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layer - on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - OPT_START_DOCSTRING, -) -# Copied from transformers.models.bart.modeling_bart.BartForQuestionAnswering with BART->OPT,Bart->OPT -class OPTForQuestionAnswering(OPTPretrainedModel): - def __init__(self, config): - super().__init__(config) - - config.num_labels = 2 - self.num_labels = config.num_labels - - self.model = OPTModel(config) - self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels) - - self.model._init_weights(self.qa_outputs) - - @add_start_docstrings_to_model_forward(OPT_INPUTS_DOCSTRING) - @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, - checkpoint=_CHECKPOINT_FOR_QA, - output_type=Seq2SeqQuestionAnsweringModelOutput, - config_class=_CONFIG_FOR_DOC, - expected_loss=_QA_EXPECTED_LOSS, - expected_output=_QA_EXPECTED_OUTPUT, - ) - def forward( - self, - input_ids: torch.Tensor = None, - attention_mask: Optional[torch.Tensor] = None, - decoder_input_ids: Optional[torch.LongTensor] = None, - decoder_attention_mask: Optional[torch.LongTensor] = None, - head_mask: Optional[torch.Tensor] = None, - decoder_head_mask: Optional[torch.Tensor] = None, - cross_attn_head_mask: Optional[torch.Tensor] = None, - encoder_outputs: Optional[List[torch.FloatTensor]] = None, - start_positions: Optional[torch.LongTensor] = None, - end_positions: Optional[torch.LongTensor] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - decoder_inputs_embeds: Optional[torch.FloatTensor] = None, - use_cache: Optional[bool] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - ) -> Union[Tuple, Seq2SeqQuestionAnsweringModelOutput]: - r""" - start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the start of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence - are not taken into account for computing the loss. - end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): - Labels for position (index) of the end of the labelled span for computing the token classification loss. - Positions are clamped to the length of the sequence (*sequence_length*). Position outside of the sequence - are not taken into account for computing the loss. - """ - return_dict = return_dict if return_dict is not None else self.config.use_return_dict - if start_positions is not None and end_positions is not None: - use_cache = False - - outputs = self.model( - input_ids, - attention_mask=attention_mask, - decoder_input_ids=decoder_input_ids, - decoder_attention_mask=decoder_attention_mask, - head_mask=head_mask, - decoder_head_mask=decoder_head_mask, - cross_attn_head_mask=cross_attn_head_mask, - encoder_outputs=encoder_outputs, - inputs_embeds=inputs_embeds, - decoder_inputs_embeds=decoder_inputs_embeds, - use_cache=use_cache, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - ) - - sequence_output = outputs[0] - - logits = self.qa_outputs(sequence_output) - start_logits, end_logits = logits.split(1, dim=-1) - start_logits = start_logits.squeeze(-1).contiguous() - end_logits = end_logits.squeeze(-1).contiguous() - - total_loss = None - if start_positions is not None and end_positions is not None: - # If we are on multi-GPU, split add a dimension - if len(start_positions.size()) > 1: - start_positions = start_positions.squeeze(-1) - if len(end_positions.size()) > 1: - end_positions = end_positions.squeeze(-1) - # sometimes the start/end positions are outside our model inputs, we ignore these terms - ignored_index = start_logits.size(1) - start_positions = start_positions.clamp(0, ignored_index) - end_positions = end_positions.clamp(0, ignored_index) - - loss_fct = CrossEntropyLoss(ignore_index=ignored_index) - start_loss = loss_fct(start_logits, start_positions) - end_loss = loss_fct(end_logits, end_positions) - total_loss = (start_loss + end_loss) / 2 - - if not return_dict: - output = ( - start_logits, - end_logits, - ) + outputs[1:] - return ((total_loss,) + output) if total_loss is not None else output - - return Seq2SeqQuestionAnsweringModelOutput( - loss=total_loss, - start_logits=start_logits, - end_logits=end_logits, - past_key_values=outputs.past_key_values, - decoder_hidden_states=outputs.decoder_hidden_states, - decoder_attentions=outputs.decoder_attentions, - cross_attentions=outputs.cross_attentions, - encoder_last_hidden_state=outputs.encoder_last_hidden_state, - encoder_hidden_states=outputs.encoder_hidden_states, - encoder_attentions=outputs.encoder_attentions, - ) - # Copied from transformers.models.bart.modeling_bart.BartDecoderWrapper with Bart->OPT class OPTDecoderWrapper(OPTPretrainedModel): @@ -1484,7 +991,6 @@ def forward(self, *args, **kwargs): return self.decoder(*args, **kwargs) -# Copied from transformers.models.bart.modeling_bart.BartForCausalLM with Bart->OPT,facebook/bart-base-> class OPTForCausalLM(OPTPretrainedModel): def __init__(self, config): config = copy.deepcopy(config) @@ -1556,13 +1062,13 @@ def forward( encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + head_mask (`torch.Tensor` of shape `(num_layers, num_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. - cross_attn_head_mask (`torch.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): + cross_attn_head_mask (`torch.Tensor` of shape `(num_layers, num_attention_heads)`, *optional*): Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, diff --git a/src/transformers/models/opt/modeling_tf_opt.py b/src/transformers/models/opt/modeling_tf_opt.py index cbf694bac78bc..ef41fc55186c0 100644 --- a/src/transformers/models/opt/modeling_tf_opt.py +++ b/src/transformers/models/opt/modeling_tf_opt.py @@ -455,40 +455,6 @@ def call( present_key_value, ) - -# Copied from transformers.models.bart.modeling_tf_bart.TFBartPretrainedModel with Bart->OPT -class TFOPTPretrainedModel(TFPreTrainedModel): - config_class = OPTConfig - base_model_prefix = "model" - - @property - def dummy_inputs(self): - pad_token = 1 - input_ids = tf.cast(tf.convert_to_tensor(DUMMY_INPUTS), tf.int32) - decoder_input_ids = tf.cast(tf.convert_to_tensor(DUMMY_INPUTS), tf.int32) - dummy_inputs = { - "decoder_input_ids": decoder_input_ids, - "attention_mask": tf.math.not_equal(input_ids, pad_token), - "input_ids": input_ids, - } - return dummy_inputs - - @tf.function( - input_signature=[ - { - "input_ids": tf.TensorSpec((None, None), tf.int32, name="input_ids"), - "attention_mask": tf.TensorSpec((None, None), tf.int32, name="attention_mask"), - "decoder_input_ids": tf.TensorSpec((None, None), tf.int32, name="decoder_input_ids"), - "decoder_attention_mask": tf.TensorSpec((None, None), tf.int32, name="decoder_attention_mask"), - } - ] - ) - def serving(self, inputs): - output = self.call(inputs) - - return self.serving_output(output) - - OPT_START_DOCSTRING = r""" This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads @@ -525,6 +491,47 @@ def serving(self, inputs): configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights. """ +@add_start_docstrings( + "The bare OPT Model outputting raw hidden-states without any specific head on top.", + OPT_START_DOCSTRING, +) +class TFOPTPretrainedModel(TFPreTrainedModel): + """ + TFOPT Pretrained Model that inheritates from transformers.TFPreTrainedModel + + Args: + config: OPTConfig + """ + config_class = OPTConfig + base_model_prefix = "model" + + @property + def dummy_inputs(self): + pad_token = 1 + input_ids = tf.cast(tf.convert_to_tensor(DUMMY_INPUTS), tf.int32) + decoder_input_ids = tf.cast(tf.convert_to_tensor(DUMMY_INPUTS), tf.int32) + dummy_inputs = { + "decoder_input_ids": decoder_input_ids, + "attention_mask": tf.math.not_equal(input_ids, pad_token), + "input_ids": input_ids, + } + return dummy_inputs + + @tf.function( + input_signature=[ + { + "input_ids": tf.TensorSpec((None, None), tf.int32, name="input_ids"), + "attention_mask": tf.TensorSpec((None, None), tf.int32, name="attention_mask"), + "decoder_input_ids": tf.TensorSpec((None, None), tf.int32, name="decoder_input_ids"), + "decoder_attention_mask": tf.TensorSpec((None, None), tf.int32, name="decoder_attention_mask"), + } + ] + ) + def serving(self, inputs): + output = self.call(inputs) + + return self.serving_output(output) + OPT_GENERATION_EXAMPLE = r""" Summarization example: @@ -1129,7 +1136,7 @@ def call( @add_start_docstrings( - "The bare OPT Model outputting raw hidden-states without any specific head on top.", + "The bare TF OPT Model outputting raw hidden-states without any specific head on top.", OPT_START_DOCSTRING, ) # Copied from transformers.models.bart.modeling_tf_bart.TFBartModel with BART->OPT,Bart->OPT diff --git a/src/transformers/utils/dummy_flax_objects.py b/src/transformers/utils/dummy_flax_objects.py index a6c6e7926da1b..e3723abacd841 100644 --- a/src/transformers/utils/dummy_flax_objects.py +++ b/src/transformers/utils/dummy_flax_objects.py @@ -795,6 +795,55 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["flax"]) +class FlaxOPTDecoderPreTrainedModel(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + +class FlaxOPTForCausalLM(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + +class FlaxOPTForConditionalGeneration(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + +class FlaxOPTForQuestionAnswering(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + +class FlaxOPTForSequenceClassification(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + +class FlaxOPTModel(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + +class FlaxOPTPreTrainedModel(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + class FlaxPegasusForConditionalGeneration(metaclass=DummyObject): _backends = ["flax"] diff --git a/src/transformers/utils/dummy_pt_objects.py b/src/transformers/utils/dummy_pt_objects.py index 112759671bbfe..5d0c395c85116 100644 --- a/src/transformers/utils/dummy_pt_objects.py +++ b/src/transformers/utils/dummy_pt_objects.py @@ -2938,6 +2938,58 @@ def load_tf_weights_in_openai_gpt(*args, **kwargs): requires_backends(load_tf_weights_in_openai_gpt, ["torch"]) +OPT_PRETRAINED_MODEL_ARCHIVE_LIST = None + + +class OPTForCausalLM(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class OPTForConditionalGeneration(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class OPTForQuestionAnswering(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class OPTForSequenceClassification(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class OPTModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class OPTPretrainedModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class PretrainedOPTModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + class PegasusForCausalLM(metaclass=DummyObject): _backends = ["torch"] diff --git a/src/transformers/utils/dummy_tf_objects.py b/src/transformers/utils/dummy_tf_objects.py index cbd3358ac8ffa..4166a896f4907 100644 --- a/src/transformers/utils/dummy_tf_objects.py +++ b/src/transformers/utils/dummy_tf_objects.py @@ -1588,6 +1588,27 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["tf"]) +class TFOPTForConditionalGeneration(metaclass=DummyObject): + _backends = ["tf"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["tf"]) + + +class TFOPTModel(metaclass=DummyObject): + _backends = ["tf"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["tf"]) + + +class TFOPTPretrainedModel(metaclass=DummyObject): + _backends = ["tf"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["tf"]) + + class TFPegasusForConditionalGeneration(metaclass=DummyObject): _backends = ["tf"] diff --git a/src/transformers/utils/dummy_tokenizers_objects.py b/src/transformers/utils/dummy_tokenizers_objects.py index 12cec6a4a2605..9971577e8089f 100644 --- a/src/transformers/utils/dummy_tokenizers_objects.py +++ b/src/transformers/utils/dummy_tokenizers_objects.py @@ -241,6 +241,13 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["tokenizers"]) +class OPTTokenizerFast(metaclass=DummyObject): + _backends = ["tokenizers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["tokenizers"]) + + class PegasusTokenizerFast(metaclass=DummyObject): _backends = ["tokenizers"] diff --git a/tests/models/opt/test_embeddings_opt.py b/tests/models/opt/test_embeddings_opt.py deleted file mode 100644 index 8d3f11903edab..0000000000000 --- a/tests/models/opt/test_embeddings_opt.py +++ /dev/null @@ -1,57 +0,0 @@ -# coding=utf-8 -# Copyright 2021, The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" Testing suite for the PyTorch OPT model. """ - -import unittest - -from transformers import BartTokenizerFast, is_torch_available - - -if is_torch_available(): - import torch - - from transformers import OPTForCausalLM - - -class OPTEmbeddingsTest(unittest.TestCase): - def setUp(self): - super().setUp() - self.path_model = "/home/younes/Desktop/Work/data/opt-350-m/" - self.path_logits_meta = "/home/younes/Desktop/Work/metaseq-conversion/logits_metaseq.p" - - def test_load_model(self): - try: - _ = OPTForCausalLM.from_pretrained(self.path_model) - except BaseException: - self.fail("Failed loading model") - - def test_logits(self): - model = OPTForCausalLM.from_pretrained(self.path_model) - model = model.eval() - tokenizer = BartTokenizerFast.from_pretrained("facebook/bart-large") - prompts = [ - "Today is a beautiful day and I want to", - "In the city of", - "Paris is the capital of France and", - "Computers and mobile phones have taken", - ] - input_ids = tokenizer(prompts, return_tensors="pt", padding=True, truncation=True).input_ids - logits = model(input_ids)[0] - logits_meta = torch.load(self.path_logits_meta) - assert torch.allclose(logits, logits_meta.permute(1, 0, 2)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index 3344fc6f37134..85c31c9672b75 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -21,7 +21,7 @@ import timeout_decorator # noqa -from transformers import OPTConfig, GPT2Tokenizer, is_torch_available, pipeline +from transformers import GPT2Tokenizer, OPTConfig, is_torch_available, pipeline from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device from transformers.utils import cached_property @@ -41,7 +41,6 @@ OPTForSequenceClassification, OPTModel, OPTTokenizer, - pipeline, ) from transformers.models.opt.modeling_opt import OPTDecoder, OPTEncoder, shift_tokens_right @@ -967,11 +966,12 @@ def test_retain_grad_hidden_states_attentions(self): # decoder cannot keep gradients return + class OPTEmbeddingsTest(unittest.TestCase): def setUp(self): super().setUp() - self.path_model = "/home/younes/Desktop/Work/data/opt-350-m/" # TODO add the model on the hub - self.path_logits_meta = "/home/younes/Desktop/Work/metaseq-conversion/logits_metaseq_gpt2_tokenizer.p" # TODO add the logits somewhere? + self.path_model = "/home/younes/Desktop/Work/data/opt-350-m/" # TODO add the model on the hub + self.path_logits_meta = "/home/younes/Desktop/Work/metaseq-conversion/logits_metaseq_gpt2_tokenizer.p" # TODO add the logits somewhere? @unittest.skip("Skipping unless we find a solution to store the model") def test_load_model(self): @@ -986,8 +986,8 @@ def test_logits(self): model = model.eval() # tokenizer = GPT2Tokenizer.from_pretrained("patrickvonplaten/opt_gpt2_tokenizer") tokenizer = GPT2Tokenizer.from_pretrained("patrickvonplaten/opt_gpt2_tokenizer") - tokenizer.add_special_tokens({'pad_token': ''}) - + tokenizer.add_special_tokens({"pad_token": ""}) + prompts = [ "Today is a beautiful day and I want to", "In the city of", @@ -999,12 +999,12 @@ def test_logits(self): logits_meta = torch.load(self.path_logits_meta) assert torch.allclose(logits, logits_meta) + class OPTGenerationTest(unittest.TestCase): def setUp(self): super().setUp() - self.path_model = "/home/younes/Desktop/Work/data/opt-350-m/" # TODO add the model on the hub - self.path_logits_meta = "/home/younes/Desktop/Work/metaseq-conversion/logits_metaseq_gpt2_tokenizer.p" # TODO add the logits somewhere? - + self.path_model = "/home/younes/Desktop/Work/data/opt-350-m/" # TODO add the model on the hub + self.path_logits_meta = "/home/younes/Desktop/Work/metaseq-conversion/logits_metaseq_gpt2_tokenizer.p" # TODO add the logits somewhere? @unittest.skip("Skipping unless we find a solution to store the model") def test_generation(self): @@ -1012,11 +1012,11 @@ def test_generation(self): model = model.eval() # tokenizer = GPT2Tokenizer.from_pretrained("patrickvonplaten/opt_gpt2_tokenizer") tokenizer = GPT2Tokenizer.from_pretrained("patrickvonplaten/opt_gpt2_tokenizer") - #tokenizer.add_special_tokens({'pad_token': ''}) + # tokenizer.add_special_tokens({'pad_token': ''}) model.config.eos_token_id = tokenizer.eos_token_id gen = pipeline("text-generation", model=model, tokenizer=tokenizer, return_tensors=True) - + prompts = [ "Today is a beautiful day and I want to", "In the city of", @@ -1026,12 +1026,12 @@ def test_generation(self): NEXT_TOKENS = [ 3392, 764, - 5, + 5, 81, ] GEN_OUTPUT = [] for prompt in prompts: len_input_sentence = len(tokenizer.tokenize(prompt)) - predicted_next_token = gen(prompt)[0]['generated_token_ids'][len_input_sentence] + predicted_next_token = gen(prompt)[0]["generated_token_ids"][len_input_sentence] GEN_OUTPUT.append(predicted_next_token) - self.assertListEqual(GEN_OUTPUT, NEXT_TOKENS) \ No newline at end of file + self.assertListEqual(GEN_OUTPUT, NEXT_TOKENS) diff --git a/utils/check_repo.py b/utils/check_repo.py index 5afaac02a123a..a46cb454c527c 100644 --- a/utils/check_repo.py +++ b/utils/check_repo.py @@ -92,6 +92,8 @@ "SeparableConv1D", # Building part of bigger (tested) model. "FlaxBartForCausalLM", # Building part of bigger (tested) model. "FlaxBertForCausalLM", # Building part of bigger (tested) model. Tested implicitly through FlaxRobertaForCausalLM. + "FlaxOPTForCausalLM", # + "OPTDecoderWrapper", ] # Update this list with test files that don't have a tester with a `all_model_classes` variable and which don't From 99001d3aacfb969175a04c231bd674cfb6fa2182 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Thu, 5 May 2022 15:39:58 +0200 Subject: [PATCH 012/145] fix code quality --- src/transformers/models/opt/modeling_opt.py | 16 ++++++---------- src/transformers/models/opt/modeling_tf_opt.py | 3 +++ utils/check_repo.py | 2 +- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index fd0f4491c74b1..9337878d8d99a 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -16,29 +16,24 @@ import copy import math import random -import warnings from typing import Dict, List, Optional, Tuple, Union import torch import torch.nn.functional as F import torch.utils.checkpoint from torch import Tensor, nn -from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss +from torch.nn import CrossEntropyLoss from ...activations import ACT2FN from ...modeling_outputs import ( BaseModelOutput, BaseModelOutputWithPastAndCrossAttentions, CausalLMOutputWithCrossAttentions, - Seq2SeqLMOutput, Seq2SeqModelOutput, - Seq2SeqQuestionAnsweringModelOutput, - Seq2SeqSequenceClassifierOutput, ) from ...modeling_utils import PreTrainedModel from ...utils import ( add_code_sample_docstrings, - add_end_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging, @@ -153,9 +148,7 @@ def forward( positions: Optional[Tensor] = None, ): """Input is expected to be of size [bsz x seqlen].""" - if not ((positions is None) or ( - self.padding_idx is None - )): + if not ((positions is None) or (self.padding_idx is None)): raise ValueError("If positions is pre-computed then padding_idx should not be set.") # we cannot use incremental state here because we must be aware of @@ -428,6 +421,7 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: hidden_states = self.out_proj(hidden_states) return hidden_states + OPT_START_DOCSTRING = r""" This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads @@ -444,6 +438,7 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: [`~PreTrainedModel.from_pretrained`] method to load the model weights. """ + @add_start_docstrings( "The bare OPT Model outputting raw hidden-states without any specific head on top.", OPT_START_DOCSTRING, @@ -464,6 +459,7 @@ def _init_weights(self, module): module.weight.data.normal_(mean=0.0, std=std) if module.padding_idx is not None: module.weight.data[module.padding_idx].zero_() + def _set_gradient_checkpointing(self, module, value=False): if isinstance(module, (OPTDecoder)): module.gradient_checkpointing = value @@ -479,6 +475,7 @@ def dummy_inputs(self): } return dummy_inputs + OPT_GENERATION_EXAMPLE = r""" Generation example: @@ -975,7 +972,6 @@ def forward( ) - # Copied from transformers.models.bart.modeling_bart.BartDecoderWrapper with Bart->OPT class OPTDecoderWrapper(OPTPretrainedModel): """ diff --git a/src/transformers/models/opt/modeling_tf_opt.py b/src/transformers/models/opt/modeling_tf_opt.py index ef41fc55186c0..d5f733a9f11d6 100644 --- a/src/transformers/models/opt/modeling_tf_opt.py +++ b/src/transformers/models/opt/modeling_tf_opt.py @@ -455,6 +455,7 @@ def call( present_key_value, ) + OPT_START_DOCSTRING = r""" This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads @@ -491,6 +492,7 @@ def call( configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights. """ + @add_start_docstrings( "The bare OPT Model outputting raw hidden-states without any specific head on top.", OPT_START_DOCSTRING, @@ -502,6 +504,7 @@ class TFOPTPretrainedModel(TFPreTrainedModel): Args: config: OPTConfig """ + config_class = OPTConfig base_model_prefix = "model" diff --git a/utils/check_repo.py b/utils/check_repo.py index a46cb454c527c..ecee201f0bce2 100644 --- a/utils/check_repo.py +++ b/utils/check_repo.py @@ -92,7 +92,7 @@ "SeparableConv1D", # Building part of bigger (tested) model. "FlaxBartForCausalLM", # Building part of bigger (tested) model. "FlaxBertForCausalLM", # Building part of bigger (tested) model. Tested implicitly through FlaxRobertaForCausalLM. - "FlaxOPTForCausalLM", # + "FlaxOPTForCausalLM", # "OPTDecoderWrapper", ] From a777bbcc1c1ae79990338d1dcfafe370812f9984 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Fri, 6 May 2022 09:32:08 +0200 Subject: [PATCH 013/145] major changes - change attention architecture to BART-like - modify some tests - style fix --- .../models/opt/configuration_opt.py | 2 +- src/transformers/models/opt/modeling_opt.py | 328 ++++++++------- tests/models/opt/test_modeling_opt.py | 383 ++++++------------ 3 files changed, 285 insertions(+), 428 deletions(-) diff --git a/src/transformers/models/opt/configuration_opt.py b/src/transformers/models/opt/configuration_opt.py index 7d341439006c1..a10fbcd18e3fc 100644 --- a/src/transformers/models/opt/configuration_opt.py +++ b/src/transformers/models/opt/configuration_opt.py @@ -95,7 +95,7 @@ class OPTConfig(PretrainedConfig): ```""" model_type = "opt" keys_to_ignore_at_inference = ["past_key_values"] - attribute_map = {"hidden_size": "d_model"} + attribute_map = {"hidden_size": "d_model", "num_hidden_layers": "num_layers"} def __init__( self, diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 9337878d8d99a..8a0013f38d517 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -26,7 +26,6 @@ from ...activations import ACT2FN from ...modeling_outputs import ( - BaseModelOutput, BaseModelOutputWithPastAndCrossAttentions, CausalLMOutputWithCrossAttentions, Seq2SeqModelOutput, @@ -205,104 +204,155 @@ def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): def forward( self, hidden_states: torch.Tensor, + key_value_states: Optional[torch.Tensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, attention_mask: Optional[torch.Tensor] = None, - self_attn_padding_mask: Optional[Tensor] = None, + layer_head_mask: Optional[torch.Tensor] = None, + key_padding_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: """Input shape: Batch x Time x Channel""" - attention_mask = attention_mask[0, 0, :, :] - - return F.multi_head_attention_forward( - hidden_states, - hidden_states, - hidden_states, - self.embed_dim, - self.num_heads, - torch.empty([0]), - torch.cat((self.q_proj.bias, self.k_proj.bias, self.v_proj.bias)), - None, - None, - self.add_zero_attn, - self.dropout, - self.out_proj.weight, - self.out_proj.bias, - self.training, - self_attn_padding_mask, - False, - attention_mask, - use_separate_proj_weight=True, - q_proj_weight=self.q_proj.weight, - k_proj_weight=self.k_proj.weight, - v_proj_weight=self.v_proj.weight, - ) - + # if key_value_states are provided this layer is used as a cross-attention layer + # for the decoder + # hidden_states = hidden_states.permute(1, 0, 2) + + is_cross_attention = key_value_states is not None + + bsz, tgt_len, _ = hidden_states.size() + + # get query proj + query_states = self.q_proj(hidden_states) * self.scaling + # get key, value proj + if is_cross_attention and past_key_value is not None: + # reuse k,v, cross_attentions + key_states = past_key_value[0] + value_states = past_key_value[1] + elif is_cross_attention: + # cross_attentions + key_states = self._shape(self.k_proj(key_value_states), -1, bsz) + value_states = self._shape(self.v_proj(key_value_states), -1, bsz) + elif past_key_value is not None: + # reuse k, v, self_attention + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + key_states = torch.cat([past_key_value[0], key_states], dim=2) + value_states = torch.cat([past_key_value[1], value_states], dim=2) + else: + # self_attention + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + + if self.is_decoder: + # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. + # Further calls to cross_attention layer can then reuse all cross-attention + # key/value_states (first "if" case) + # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of + # all previous decoder key/value_states. Further calls to uni-directional self-attention + # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) + # if encoder bi-directional self-attention `past_key_value` is always `None` + past_key_value = (key_states, value_states) + + proj_shape = (bsz * self.num_heads, -1, self.head_dim) + query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) + key_states = key_states.view(*proj_shape) + value_states = value_states.view(*proj_shape) + + src_len = key_states.size(1) -class OPTEncoderLayer(nn.Module): - def __init__(self, config: OPTConfig): - super().__init__() - self.embed_dim = config.d_model - self.self_attn = OPTAttention( - embed_dim=self.embed_dim, - num_heads=config.encoder_attention_heads, - dropout=config.attention_dropout, - ) - self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim) - self.dropout = config.dropout - self.activation_fn = ACT2FN[config.activation_function] - self.activation_dropout = config.activation_dropout - self.fc1 = nn.Linear(self.embed_dim, config.encoder_ffn_dim) - self.fc2 = nn.Linear(config.encoder_ffn_dim, self.embed_dim) - self.final_layer_norm = nn.LayerNorm(self.embed_dim) + if attention_mask is not None: + if attention_mask.dtype == torch.uint8: + # warnings.warn("Byte tensor for attn_mask in nn.MultiheadAttention is deprecated. Use bool tensor instead.") + attention_mask = attention_mask.to(torch.bool) + else: + assert ( + attention_mask.is_floating_point() or attention_mask.dtype == torch.bool + ), f"Only float, byte, and bool types are supported for attn_mask, not {attention_mask.dtype}" + # ensure attn_mask's dim is 3 + if attention_mask.dim() == 2: + correct_2d_size = (tgt_len, src_len) + if attention_mask.shape != correct_2d_size: + raise RuntimeError( + f"The shape of the 2D attn_mask is {attention_mask.shape}, but should be {correct_2d_size}." + ) + attention_mask = attention_mask.unsqueeze(0) + elif attention_mask.dim() == 3: + correct_3d_size = (bsz * self.num_heads, tgt_len, src_len) + if attention_mask.shape != correct_3d_size: + raise RuntimeError( + f"The shape of the 3D attn_mask is {attention_mask.shape}, but should be {correct_3d_size}." + ) + elif attention_mask.dim() == 4: + attention_mask = attention_mask[0, 0, :, :] # consider only one mask since they should be all the same + else: + raise RuntimeError(f"attn_mask's dimension {attention_mask.dim()} is not supported") + + if key_padding_mask is not None: + assert key_padding_mask.shape == ( + bsz, + src_len, + ), f"expecting key_padding_mask shape of {(bsz, src_len)}, but got {key_padding_mask.shape}" + key_padding_mask = ( + key_padding_mask.view(bsz, 1, 1, src_len) + .expand(-1, self.num_heads, -1, -1) + .reshape(bsz * self.num_heads, 1, src_len) + ) + if attention_mask is None: + attention_mask = key_padding_mask + elif attention_mask.dtype == torch.bool: + attention_mask = attention_mask.logical_or(key_padding_mask) + else: + attention_mask = attention_mask.masked_fill(key_padding_mask, float("-inf")) - def forward( - self, - hidden_states: torch.FloatTensor, - attention_mask: torch.FloatTensor, - layer_head_mask: torch.FloatTensor, - output_attentions: Optional[bool] = False, - ) -> Tuple[torch.FloatTensor, Optional[torch.FloatTensor]]: - """ - Args: - hidden_states (`torch.FloatTensor`): input to the layer of shape `(seq_len, batch, embed_dim)` - attention_mask (`torch.FloatTensor`): attention mask of size - `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. - layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size - `(encoder_attention_heads,)`. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - """ - residual = hidden_states - hidden_states, attn_weights, _ = self.self_attn( - hidden_states=hidden_states, - attention_mask=attention_mask, - layer_head_mask=layer_head_mask, - output_attentions=output_attentions, - ) - hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) - hidden_states = residual + hidden_states - hidden_states = self.self_attn_layer_norm(hidden_states) + if attention_mask is not None: + attn_weights = torch.baddbmm(attention_mask, query_states, key_states.transpose(-2, -1)) + else: + attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) - residual = hidden_states - # hidden_states = self.activation_fn(self.fc1(hidden_states)) - hidden_states = nn.functional.dropout(hidden_states, p=self.activation_dropout, training=self.training) - hidden_states = self.fc2(hidden_states) - hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) - hidden_states = residual + hidden_states - hidden_states = self.final_layer_norm(hidden_states) + if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): + raise ValueError( + f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is {attn_weights.size()}" + ) - if hidden_states.dtype == torch.float16 and ( - torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any() - ): - clamp_value = torch.finfo(hidden_states.dtype).max - 1000 - hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value) + attn_weights = nn.functional.softmax(attn_weights, dim=-1) - outputs = (hidden_states,) + if layer_head_mask is not None: + if layer_head_mask.size() != (self.num_heads,): + raise ValueError( + f"Head mask for a single layer should be of size {(self.num_heads,)}, but is {layer_head_mask.size()}" + ) + attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) if output_attentions: - outputs += (attn_weights,) + # this operation is a bit awkward, but it's required to + # make sure that attn_weights keeps its gradient. + # In order to do so, attn_weights have to be reshaped + # twice and have to be reused in the following + attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) + else: + attn_weights_reshaped = None - return outputs + attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + + attn_output = torch.bmm(attn_probs, value_states) + + if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is {attn_output.size()}" + ) + + attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim) + attn_output = attn_output.transpose(1, 2) + + # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be + # partitioned aross GPUs when using tensor-parallelism. + attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim) + + attn_output = self.out_proj(attn_output) + + return attn_output, attn_weights_reshaped, past_key_value class OPTDecoderLayer(nn.Module): @@ -332,20 +382,16 @@ def forward( hidden_states: torch.Tensor, attention_mask: Optional[torch.Tensor] = None, layer_head_mask: Optional[torch.Tensor] = None, - past_key_value: Optional[Tuple[torch.Tensor]] = None, output_attentions: Optional[bool] = False, use_cache: Optional[bool] = False, - self_attn_padding_mask: Optional[Tensor] = None, + key_padding_mask: Optional[Tensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: """ Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` attention_mask (`torch.FloatTensor`): attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. - encoder_hidden_states (`torch.FloatTensor`): - cross attention input to the layer of shape `(batch, seq_len, embed_dim)` - encoder_attention_mask (`torch.FloatTensor`): encoder attention mask of size - `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size `(encoder_attention_heads,)`. cross_attn_layer_head_mask (`torch.FloatTensor`): mask for cross-attention heads in a given layer of @@ -362,10 +408,21 @@ def forward( # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 # self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None # add present self-attn cache to positions 1,2 of present_key_value tuple - hidden_states, self_attn_weights = self.self_attn( + # hidden_states, self_attn_weights, present_key_value = self.self_attn( + # hidden_states=hidden_states, + # attention_mask=attention_mask, + # self_attn_padding_mask=self_attn_padding_mask, + # layer_head_mask=layer_head_mask, + # output_attentions=output_attentions, + # past_key_value=past_key_value, + # ) + hidden_states, self_attn_weights, present_key_value = self.self_attn( hidden_states=hidden_states, + past_key_value=past_key_value, attention_mask=attention_mask, - self_attn_padding_mask=self_attn_padding_mask, + layer_head_mask=layer_head_mask, + output_attentions=output_attentions, + key_padding_mask=key_padding_mask, ) hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) hidden_states = residual + hidden_states @@ -392,8 +449,8 @@ def forward( if output_attentions: outputs += (self_attn_weights, cross_attn_weights) - # if use_cache: - # outputs += (present_key_value,) + if use_cache: + outputs += (present_key_value,) return outputs @@ -447,7 +504,7 @@ class OPTPretrainedModel(PreTrainedModel): config_class = OPTConfig base_model_prefix = "model" supports_gradient_checkpointing = True - _keys_to_ignore_on_load_unexpected = [r"encoder\.version", r"decoder\.version"] + _keys_to_ignore_on_load_unexpected = [r"decoder\.version"] def _init_weights(self, module): std = self.config.init_std @@ -528,10 +585,6 @@ def dummy_inputs(self): - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. - encoder_outputs (`tuple(tuple(torch.FloatTensor)`, *optional*): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape @@ -634,8 +687,6 @@ def forward( self, input_ids: torch.LongTensor = None, attention_mask: Optional[torch.Tensor] = None, - encoder_hidden_states: Optional[torch.FloatTensor] = None, - encoder_attention_mask: Optional[torch.LongTensor] = None, head_mask: Optional[torch.Tensor] = None, self_attn_padding_mask: Optional[torch.Tensor] = None, cross_attn_head_mask: Optional[torch.Tensor] = None, @@ -662,17 +713,6 @@ def forward( - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention - of the decoder. - encoder_attention_mask (`torch.LongTensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): - Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values - selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) head_mask (`torch.Tensor` of shape `(num_layers, num_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: @@ -690,7 +730,6 @@ def forward( past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of - shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. @@ -744,11 +783,6 @@ def forward( if self_attn_padding_mask is None and (self.cross_self_attention or input_ids.eq(self.padding_idx).any()): self_attn_padding_mask = input_ids.eq(self.padding_idx) - # expand encoder attention mask - if encoder_hidden_states is not None and encoder_attention_mask is not None: - # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] - encoder_attention_mask = _expand_mask(encoder_attention_mask, inputs_embeds.dtype, tgt_len=input_shape[-1]) - # embed positions positions = self.embed_positions(input_ids) @@ -758,12 +792,12 @@ def forward( hidden_states = inputs_embeds + positions hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) - hidden_states = hidden_states.permute(1, 0, 2) + # hidden_states = hidden_states.permute(1, 0, 2) # decoder layers all_hidden_states = () if output_hidden_states else None all_self_attns = () if output_attentions else None - all_cross_attentions = () if (output_attentions and encoder_hidden_states is not None) else None + all_cross_attentions = () if (output_attentions is not None) else None next_decoder_cache = () if use_cache else None # check if head_mask/cross_attn_head_mask has a correct number of layers specified if desired @@ -803,8 +837,6 @@ def custom_forward(*inputs): create_custom_forward(decoder_layer), hidden_states, attention_mask, - encoder_hidden_states, - encoder_attention_mask, head_mask[idx] if head_mask is not None else None, cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None, None, @@ -818,19 +850,16 @@ def custom_forward(*inputs): past_key_value=past_key_value, output_attentions=output_attentions, use_cache=use_cache, - self_attn_padding_mask=self_attn_padding_mask, + key_padding_mask=self_attn_padding_mask, ) hidden_states = layer_outputs[0] - # if use_cache: - # next_decoder_cache += (layer_outputs[3 if output_attentions else 1],) + if use_cache: + next_decoder_cache += (layer_outputs[3 if output_attentions else 1],) if output_attentions: all_self_attns += (layer_outputs[1],) - if encoder_hidden_states is not None: - all_cross_attentions += (layer_outputs[2],) - if self.project_out_dim is not None: hidden_states = self.project_out_dim(hidden_states) @@ -892,7 +921,6 @@ def forward( head_mask: Optional[torch.Tensor] = None, decoder_head_mask: Optional[torch.Tensor] = None, cross_attn_head_mask: Optional[torch.Tensor] = None, - encoder_outputs: Optional[List[torch.FloatTensor]] = None, past_key_values: Optional[List[torch.FloatTensor]] = None, inputs_embeds: Optional[torch.FloatTensor] = None, decoder_inputs_embeds: Optional[torch.FloatTensor] = None, @@ -923,30 +951,12 @@ def forward( use_cache = use_cache if use_cache is not None else self.config.use_cache return_dict = return_dict if return_dict is not None else self.config.use_return_dict - if encoder_outputs is None: - encoder_outputs = self.encoder( - input_ids=input_ids, - attention_mask=attention_mask, - head_mask=head_mask, - inputs_embeds=inputs_embeds, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - ) # If the user passed a tuple for encoder_outputs, we wrap it in a BaseModelOutput when return_dict=True - elif return_dict and not isinstance(encoder_outputs, BaseModelOutput): - encoder_outputs = BaseModelOutput( - last_hidden_state=encoder_outputs[0], - hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None, - attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None, - ) # decoder outputs consists of (dec_features, past_key_value, dec_hidden, dec_attn) decoder_outputs = self.decoder( input_ids=decoder_input_ids, attention_mask=decoder_attention_mask, - encoder_hidden_states=encoder_outputs[0], - encoder_attention_mask=attention_mask, head_mask=decoder_head_mask, cross_attn_head_mask=cross_attn_head_mask, past_key_values=past_key_values, @@ -957,18 +967,12 @@ def forward( return_dict=return_dict, ) - if not return_dict: - return decoder_outputs + encoder_outputs - return Seq2SeqModelOutput( last_hidden_state=decoder_outputs.last_hidden_state, past_key_values=decoder_outputs.past_key_values, decoder_hidden_states=decoder_outputs.hidden_states, decoder_attentions=decoder_outputs.attentions, cross_attentions=decoder_outputs.cross_attentions, - encoder_last_hidden_state=encoder_outputs.last_hidden_state, - encoder_hidden_states=encoder_outputs.hidden_states, - encoder_attentions=encoder_outputs.attentions, ) @@ -1023,8 +1027,6 @@ def forward( self, input_ids: torch.LongTensor = None, attention_mask: Optional[torch.Tensor] = None, - encoder_hidden_states: Optional[torch.FloatTensor] = None, - encoder_attention_mask: Optional[torch.FloatTensor] = None, head_mask: Optional[torch.Tensor] = None, cross_attn_head_mask: Optional[torch.Tensor] = None, past_key_values: Optional[List[torch.FloatTensor]] = None, @@ -1052,12 +1054,6 @@ def forward( - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) - encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention - if the model is configured as a decoder. - encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used - in the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: head_mask (`torch.Tensor` of shape `(num_layers, num_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: @@ -1140,11 +1136,13 @@ def forward( return_dict=return_dict, ) - logits = self.lm_head(outputs[0]).permute(1, 0, 2) + # logits = self.lm_head(outputs[0]).permute(1, 0, 2).contiguous() + logits = self.lm_head(outputs[0]).contiguous() loss = None if labels is not None: loss_fct = CrossEntropyLoss() + loss = loss_fct(logits.view(-1, self.config.vocab_size), labels.view(-1)) if not return_dict: diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index 85c31c9672b75..7a11ea944ce33 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -21,7 +21,7 @@ import timeout_decorator # noqa -from transformers import GPT2Tokenizer, OPTConfig, is_torch_available, pipeline +from transformers import OPTConfig, is_torch_available, pipeline from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device from transformers.utils import cached_property @@ -33,16 +33,8 @@ if is_torch_available(): import torch - from transformers import ( - AutoModelForSequenceClassification, - OPTForCausalLM, - OPTForConditionalGeneration, - OPTForQuestionAnswering, - OPTForSequenceClassification, - OPTModel, - OPTTokenizer, - ) - from transformers.models.opt.modeling_opt import OPTDecoder, OPTEncoder, shift_tokens_right + from transformers import GPT2Tokenizer, OPTForCausalLM, OPTModel + from transformers.models.opt.modeling_opt import OPTDecoder, shift_tokens_right def prepare_opt_inputs_dict( @@ -114,6 +106,7 @@ def __init__( self.eos_token_id = eos_token_id self.pad_token_id = pad_token_id self.bos_token_id = bos_token_id + self.is_encoder_decoder = False def prepare_config_and_inputs(self): input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) @@ -132,12 +125,9 @@ def get_config(self): return OPTConfig( vocab_size=self.vocab_size, d_model=self.hidden_size, - encoder_layers=self.num_hidden_layers, - decoder_layers=self.num_hidden_layers, - encoder_attention_heads=self.num_attention_heads, - decoder_attention_heads=self.num_attention_heads, - encoder_ffn_dim=self.intermediate_size, - decoder_ffn_dim=self.intermediate_size, + num_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + ffn_dim=self.intermediate_size, dropout=self.hidden_dropout_prob, attention_dropout=self.attention_probs_dropout_prob, max_position_embeddings=self.max_position_embeddings, @@ -189,37 +179,39 @@ def create_and_check_decoder_model_past_large_inputs(self, config, inputs_dict): # test that outputs are equal for slice self.parent.assertTrue(torch.allclose(output_from_past_slice, output_from_no_past_slice, atol=1e-3)) - def check_encoder_decoder_model_standalone(self, config, inputs_dict): - model = OPTModel(config=config).to(torch_device).eval() - outputs = model(**inputs_dict) + # Do not use this test since it is for encoder decoders? - encoder_last_hidden_state = outputs.encoder_last_hidden_state - last_hidden_state = outputs.last_hidden_state + # def check_encoder_decoder_model_standalone(self, config, inputs_dict): + # model = OPTModel(config=config).to(torch_device).eval() + # outputs = model(**inputs_dict) - with tempfile.TemporaryDirectory() as tmpdirname: - encoder = model.get_encoder() - encoder.save_pretrained(tmpdirname) - encoder = OPTEncoder.from_pretrained(tmpdirname).to(torch_device) + # encoder_last_hidden_state = outputs.encoder_last_hidden_state + # last_hidden_state = outputs.last_hidden_state - encoder_last_hidden_state_2 = encoder(inputs_dict["input_ids"], attention_mask=inputs_dict["attention_mask"])[ - 0 - ] + # with tempfile.TemporaryDirectory() as tmpdirname: + # encoder = model.get_encoder() + # encoder.save_pretrained(tmpdirname) + # encoder = OPTEncoder.from_pretrained(tmpdirname).to(torch_device) + + # encoder_last_hidden_state_2 = encoder(inputs_dict["input_ids"], attention_mask=inputs_dict["attention_mask"])[ + # 0 + # ] - self.parent.assertTrue((encoder_last_hidden_state_2 - encoder_last_hidden_state).abs().max().item() < 1e-3) + # self.parent.assertTrue((encoder_last_hidden_state_2 - encoder_last_hidden_state).abs().max().item() < 1e-3) - with tempfile.TemporaryDirectory() as tmpdirname: - decoder = model.get_decoder() - decoder.save_pretrained(tmpdirname) - decoder = OPTDecoder.from_pretrained(tmpdirname).to(torch_device) + # with tempfile.TemporaryDirectory() as tmpdirname: + # decoder = model.get_decoder() + # decoder.save_pretrained(tmpdirname) + # decoder = OPTDecoder.from_pretrained(tmpdirname).to(torch_device) - last_hidden_state_2 = decoder( - input_ids=inputs_dict["decoder_input_ids"], - attention_mask=inputs_dict["decoder_attention_mask"], - encoder_hidden_states=encoder_last_hidden_state, - encoder_attention_mask=inputs_dict["attention_mask"], - )[0] + # last_hidden_state_2 = decoder( + # input_ids=inputs_dict["decoder_input_ids"], + # attention_mask=inputs_dict["decoder_attention_mask"], + # encoder_hidden_states=encoder_last_hidden_state, + # encoder_attention_mask=inputs_dict["attention_mask"], + # )[0] - self.parent.assertTrue((last_hidden_state_2 - last_hidden_state).abs().max().item() < 1e-3) + # self.parent.assertTrue((last_hidden_state_2 - last_hidden_state).abs().max().item() < 1e-3) @require_torch @@ -251,12 +243,9 @@ def _get_config_and_data(self): config = OPTConfig( vocab_size=self.vocab_size, d_model=24, - encoder_layers=2, - decoder_layers=2, - encoder_attention_heads=2, - decoder_attention_heads=2, - encoder_ffn_dim=32, - decoder_ffn_dim=32, + num_layers=2, + num_attention_heads=2, + ffn_dim=32, max_position_embeddings=48, eos_token_id=2, pad_token_id=1, @@ -264,80 +253,31 @@ def _get_config_and_data(self): ) return config, input_ids, batch_size - def test_sequence_classification_forward(self): - config, input_ids, batch_size = self._get_config_and_data() - labels = _long_tensor([2] * batch_size).to(torch_device) - model = OPTForSequenceClassification(config) - model.to(torch_device) - outputs = model(input_ids=input_ids, decoder_input_ids=input_ids, labels=labels) - expected_shape = torch.Size((batch_size, config.num_labels)) - self.assertEqual(outputs["logits"].shape, expected_shape) - self.assertIsInstance(outputs["loss"].item(), float) - - def test_question_answering_forward(self): - config, input_ids, batch_size = self._get_config_and_data() - sequence_labels = ids_tensor([batch_size], 2).to(torch_device) - model = OPTForQuestionAnswering(config) - model.to(torch_device) - outputs = model( - input_ids=input_ids, - start_positions=sequence_labels, - end_positions=sequence_labels, - ) - - self.assertEqual(outputs["start_logits"].shape, input_ids.shape) - self.assertEqual(outputs["end_logits"].shape, input_ids.shape) - self.assertIsInstance(outputs["loss"].item(), float) - @timeout_decorator.timeout(1) def test_lm_forward(self): config, input_ids, batch_size = self._get_config_and_data() lm_labels = ids_tensor([batch_size, input_ids.shape[1]], self.vocab_size).to(torch_device) - lm_model = OPTForConditionalGeneration(config) + lm_model = OPTForCausalLM(config) lm_model.to(torch_device) outputs = lm_model(input_ids=input_ids, labels=lm_labels) expected_shape = (batch_size, input_ids.shape[1], config.vocab_size) self.assertEqual(outputs["logits"].shape, expected_shape) self.assertIsInstance(outputs["loss"].item(), float) - def test_lm_uneven_forward(self): - config = OPTConfig( - vocab_size=self.vocab_size, - d_model=14, - encoder_layers=2, - decoder_layers=2, - encoder_attention_heads=2, - decoder_attention_heads=2, - encoder_ffn_dim=8, - decoder_ffn_dim=8, - max_position_embeddings=48, - ) - lm_model = OPTForConditionalGeneration(config).to(torch_device) - context = torch.tensor( - [[71, 82, 18, 33, 46, 91, 2], [68, 34, 26, 58, 30, 2, 1]], device=torch_device, dtype=torch.long - ) - summary = torch.tensor([[82, 71, 82, 18, 2], [58, 68, 2, 1, 1]], device=torch_device, dtype=torch.long) - outputs = lm_model(input_ids=context, decoder_input_ids=summary, labels=summary) - expected_shape = (*summary.shape, config.vocab_size) - self.assertEqual(outputs["logits"].shape, expected_shape) - def test_generate_beam_search(self): input_ids = torch.tensor([[71, 82, 2], [68, 34, 2]], device=torch_device, dtype=torch.long) config = OPTConfig( vocab_size=self.vocab_size, d_model=24, - encoder_layers=2, - decoder_layers=2, - encoder_attention_heads=2, - decoder_attention_heads=2, - encoder_ffn_dim=32, - decoder_ffn_dim=32, + num_layers=2, + num_attention_heads=2, + ffn_dim=32, max_position_embeddings=48, eos_token_id=2, pad_token_id=1, bos_token_id=0, ) - lm_model = OPTForConditionalGeneration(config).to(torch_device) + lm_model = OPTForCausalLM(config).to(torch_device) lm_model.eval() max_length = 5 @@ -362,7 +302,7 @@ def test_shift_tokens_right(self): @slow def test_tokenization(self): - tokenizer = OPTTokenizer.from_pretrained("facebook/opt-large") + tokenizer = GPT2Tokenizer.from_pretrained("facebook/opt-large") examples = [" Hello world", " DomDramg"] # need leading spaces for equality fairseq_results = [ torch.tensor([0, 20920, 232, 2]), @@ -375,7 +315,7 @@ def test_tokenization(self): def test_generate_fp16(self): config, input_ids, batch_size = self._get_config_and_data() attention_mask = input_ids.ne(1).to(torch_device) - model = OPTForConditionalGeneration(config).eval().to(torch_device) + model = OPTForCausalLM(config).eval().to(torch_device) if torch_device == "cuda": model.half() model.generate(input_ids, attention_mask=attention_mask) @@ -383,7 +323,7 @@ def test_generate_fp16(self): def test_dummy_inputs(self): config, *_ = self._get_config_and_data() - model = OPTForConditionalGeneration(config).eval().to(torch_device) + model = OPTForCausalLM(config).eval().to(torch_device) model(**model.dummy_inputs) def test_resize_tokens_embeddings_more(self): @@ -392,7 +332,7 @@ def test_resize_tokens_embeddings_more(self): def _get_embs(m): return (m.get_input_embeddings().weight.data.clone(), m.get_output_embeddings().weight.data.clone()) - model = OPTForConditionalGeneration(config).eval().to(torch_device) + model = OPTForCausalLM(config).eval().to(torch_device) input, output = _get_embs(model) self.assertTrue(torch.eq(input, output).all()) new_vocab_size = 45 @@ -405,13 +345,9 @@ def _get_embs(m): @require_torch class OPTModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase): - all_model_classes = ( - (OPTModel, OPTForConditionalGeneration, OPTForSequenceClassification, OPTForQuestionAnswering) - if is_torch_available() - else () - ) - all_generative_model_classes = (OPTForConditionalGeneration,) if is_torch_available() else () - is_encoder_decoder = True + all_model_classes = (OPTModel, OPTForCausalLM) if is_torch_available() else () + all_generative_model_classes = (OPTForCausalLM,) if is_torch_available() else () + is_encoder_decoder = False test_pruning = False test_missing_keys = False @@ -444,7 +380,7 @@ def test_encoder_decoder_model_standalone(self): def test_inputs_embeds(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - for model_class in (OPTModel, OPTForConditionalGeneration, OPTForQuestionAnswering): + for model_class in OPTModel: model = model_class(config) model.to(torch_device) model.eval() @@ -470,15 +406,15 @@ def test_inputs_embeds(self): with torch.no_grad(): model(**inputs)[0] - def test_generate_fp16(self): - config, input_dict = self.model_tester.prepare_config_and_inputs() - input_ids = input_dict["input_ids"] - attention_mask = input_ids.ne(1).to(torch_device) - model = OPTForConditionalGeneration(config).eval().to(torch_device) - if torch_device == "cuda": - model.half() - model.generate(input_ids, attention_mask=attention_mask) - model.generate(num_beams=4, do_sample=True, early_stopping=False, num_return_sequences=3) + # def test_generate_fp16(self): + # config, input_dict = self.model_tester.prepare_config_and_inputs() + # input_ids = input_dict["input_ids"] + # attention_mask = input_ids.ne(1).to(torch_device) + # model = OPTForConditionalGeneration(config).eval().to(torch_device) + # if torch_device == "cuda": + # model.half() + # model.generate(input_ids, attention_mask=attention_mask) + # model.generate(num_beams=4, do_sample=True, early_stopping=False, num_return_sequences=3) def assert_tensors_close(a, b, atol=1e-12, prefix=""): @@ -511,11 +447,11 @@ class FastIntegrationTests(unittest.TestCase): @cached_property def tok(self): - return OPTTokenizer.from_pretrained("facebook/opt-large") + return GPT2Tokenizer.from_pretrained("facebook/opt-large") @cached_property def xsum_1_1_model(self): - return OPTForConditionalGeneration.from_pretrained("sshleifer/distilopt-xsum-1-1") + return OPTForCausalLM.from_pretrained("sshleifer/distilopt-xsum-1-1") def test_xsum_1_1_generation(self): hf = self.xsum_1_1_model @@ -574,7 +510,7 @@ def test_encoder_equiv(self): class OPTModelIntegrationTests(unittest.TestCase): @cached_property def default_tokenizer(self): - return OPTTokenizer.from_pretrained("facebook/opt-large") + return GPT2Tokenizer.from_pretrained("facebook/opt-large") @slow def test_inference_no_head(self): @@ -590,82 +526,48 @@ def test_inference_no_head(self): ) self.assertTrue(torch.allclose(output[:, :3, :3], expected_slice, atol=1e-3)) - @slow - def test_base_mask_filling(self): - pbase = pipeline(task="fill-mask", model="") - src_text = [" I went to the ."] - results = [x["token_str"] for x in pbase(src_text)] - assert " bathroom" in results - - @slow - def test_large_mask_filling(self): - plarge = pipeline(task="fill-mask", model="facebook/opt-large") - src_text = [" I went to the ."] - results = [x["token_str"] for x in plarge(src_text)] - expected_results = [" bathroom", " gym", " wrong", " movies", " hospital"] - self.assertListEqual(results, expected_results) - - @slow - def test_mnli_inference(self): - example_b = [0, 31414, 232, 328, 740, 1140, 69, 46078, 1588, 2, 1] - input_ids = _long_tensor([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2], example_b]) - - model = AutoModelForSequenceClassification.from_pretrained("facebook/opt-large-mnli").to( - torch_device - ) # eval called in from_pre - attention_mask = input_ids.ne(model.config.pad_token_id) - # Test that model hasn't changed - with torch.no_grad(): - outputs = model(input_ids=input_ids, attention_mask=attention_mask) - - batched_logits = outputs.logits - expected_shape = torch.Size((2, 3)) - self.assertEqual(batched_logits.shape, expected_shape) - expected_slice = torch.tensor([[0.1907, 1.4342, -1.0289]], device=torch_device) - logits_arr = batched_logits[0].detach() - - # Test that padding does not change results - input_ids_no_pad = _long_tensor([example_b[:-1]]) - attention_mask_no_pad = input_ids_no_pad.ne(model.config.pad_token_id) - - with torch.no_grad(): - logits2 = model(input_ids=input_ids_no_pad, attention_mask=attention_mask_no_pad).logits.squeeze() - assert_tensors_close(batched_logits[1], logits2, atol=1e-3) - assert_tensors_close(expected_slice, logits_arr, atol=1e-3) - - @slow - def test_xsum_summarization_same_as_fairseq(self): - model = OPTForConditionalGeneration.from_pretrained("facebook/opt-large-xsum").to(torch_device) - tok = self.default_tokenizer - - PGE_ARTICLE = """ PG&E stated it scheduled the blackouts in response to forecasts for high winds amid dry conditions. The aim is to reduce the risk of wildfires. Nearly 800 thousand customers were scheduled to be affected by the shutoffs which were expected to last through at least midday tomorrow.""" - - EXPECTED_SUMMARY = "California's largest power company has begun shutting off electricity to thousands of customers in the state." - dct = tok.batch_encode_plus( - [PGE_ARTICLE], - max_length=1024, - padding="max_length", - truncation=True, - return_tensors="pt", - ).to(torch_device) - - hypotheses_batch = model.generate( - input_ids=dct["input_ids"], - attention_mask=dct["attention_mask"], - num_beams=2, - max_length=62, - min_length=11, - length_penalty=1.0, - no_repeat_ngram_size=3, - early_stopping=True, - decoder_start_token_id=model.config.eos_token_id, - ) - - decoded = tok.batch_decode( - hypotheses_batch, - skip_special_tokens=True, - ) - self.assertEqual(EXPECTED_SUMMARY, decoded[0]) + # @slow + # def test_base_mask_filling(self): + # pbase = pipeline(task="fill-mask", model="") + # src_text = [" I went to the ."] + # results = [x["token_str"] for x in pbase(src_text)] + # assert " bathroom" in results + + # @slow + # def test_large_mask_filling(self): + # plarge = pipeline(task="fill-mask", model="facebook/opt-large") + # src_text = [" I went to the ."] + # results = [x["token_str"] for x in plarge(src_text)] + # expected_results = [" bathroom", " gym", " wrong", " movies", " hospital"] + # self.assertListEqual(results, expected_results) + + # @slow + # def test_mnli_inference(self): + # example_b = [0, 31414, 232, 328, 740, 1140, 69, 46078, 1588, 2, 1] + # input_ids = _long_tensor([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2], example_b]) + + # model = AutoModelForSequenceClassification.from_pretrained("facebook/opt-large-mnli").to( + # torch_device + # ) # eval called in from_pre + # attention_mask = input_ids.ne(model.config.pad_token_id) + # # Test that model hasn't changed + # with torch.no_grad(): + # outputs = model(input_ids=input_ids, attention_mask=attention_mask) + + # batched_logits = outputs.logits + # expected_shape = torch.Size((2, 3)) + # self.assertEqual(batched_logits.shape, expected_shape) + # expected_slice = torch.tensor([[0.1907, 1.4342, -1.0289]], device=torch_device) + # logits_arr = batched_logits[0].detach() + + # # Test that padding does not change results + # input_ids_no_pad = _long_tensor([example_b[:-1]]) + # attention_mask_no_pad = input_ids_no_pad.ne(model.config.pad_token_id) + + # with torch.no_grad(): + # logits2 = model(input_ids=input_ids_no_pad, attention_mask=attention_mask_no_pad).logits.squeeze() + # assert_tensors_close(batched_logits[1], logits2, atol=1e-3) + # assert_tensors_close(expected_slice, logits_arr, atol=1e-3) def test_xsum_config_generation_params(self): config = OPTConfig.from_pretrained("facebook/opt-large-xsum") @@ -673,60 +575,6 @@ def test_xsum_config_generation_params(self): config_params = {k: getattr(config, k, "MISSING") for k, v in expected_params.items()} self.assertDictEqual(expected_params, config_params) - @slow - def test_cnn_summarization_same_as_fairseq(self): - hf = OPTForConditionalGeneration.from_pretrained("facebook/opt-large-cnn").to(torch_device) - tok = OPTTokenizer.from_pretrained("facebook/opt-large") - - FRANCE_ARTICLE = ' Marseille, France (CNN)The French prosecutor leading an investigation into the crash of Germanwings Flight 9525 insisted Wednesday that he was not aware of any video footage from on board the plane. Marseille prosecutor Brice Robin told CNN that "so far no videos were used in the crash investigation." He added, "A person who has such a video needs to immediately give it to the investigators." Robin\'s comments follow claims by two magazines, German daily Bild and French Paris Match, of a cell phone video showing the harrowing final seconds from on board Germanwings Flight 9525 as it crashed into the French Alps. All 150 on board were killed. Paris Match and Bild reported that the video was recovered from a phone at the wreckage site. The two publications described the supposed video, but did not post it on their websites. The publications said that they watched the video, which was found by a source close to the investigation. "One can hear cries of \'My God\' in several languages," Paris Match reported. "Metallic banging can also be heard more than three times, perhaps of the pilot trying to open the cockpit door with a heavy object. Towards the end, after a heavy shake, stronger than the others, the screaming intensifies. Then nothing." "It is a very disturbing scene," said Julian Reichelt, editor-in-chief of Bild online. An official with France\'s accident investigation agency, the BEA, said the agency is not aware of any such video. Lt. Col. Jean-Marc Menichini, a French Gendarmerie spokesman in charge of communications on rescue efforts around the Germanwings crash site, told CNN that the reports were "completely wrong" and "unwarranted." Cell phones have been collected at the site, he said, but that they "hadn\'t been exploited yet." Menichini said he believed the cell phones would need to be sent to the Criminal Research Institute in Rosny sous-Bois, near Paris, in order to be analyzed by specialized technicians working hand-in-hand with investigators. But none of the cell phones found so far have been sent to the institute, Menichini said. Asked whether staff involved in the search could have leaked a memory card to the media, Menichini answered with a categorical "no." Reichelt told "Erin Burnett: Outfront" that he had watched the video and stood by the report, saying Bild and Paris Match are "very confident" that the clip is real. He noted that investigators only revealed they\'d recovered cell phones from the crash site after Bild and Paris Match published their reports. "That is something we did not know before. ... Overall we can say many things of the investigation weren\'t revealed by the investigation at the beginning," he said. What was mental state of Germanwings co-pilot? German airline Lufthansa confirmed Tuesday that co-pilot Andreas Lubitz had battled depression years before he took the controls of Germanwings Flight 9525, which he\'s accused of deliberately crashing last week in the French Alps. Lubitz told his Lufthansa flight training school in 2009 that he had a "previous episode of severe depression," the airline said Tuesday. Email correspondence between Lubitz and the school discovered in an internal investigation, Lufthansa said, included medical documents he submitted in connection with resuming his flight training. The announcement indicates that Lufthansa, the parent company of Germanwings, knew of Lubitz\'s battle with depression, allowed him to continue training and ultimately put him in the cockpit. Lufthansa, whose CEO Carsten Spohr previously said Lubitz was 100% fit to fly, described its statement Tuesday as a "swift and seamless clarification" and said it was sharing the information and documents -- including training and medical records -- with public prosecutors. Spohr traveled to the crash site Wednesday, where recovery teams have been working for the past week to recover human remains and plane debris scattered across a steep mountainside. He saw the crisis center set up in Seyne-les-Alpes, laid a wreath in the village of Le Vernet, closer to the crash site, where grieving families have left flowers at a simple stone memorial. Menichini told CNN late Tuesday that no visible human remains were left at the site but recovery teams would keep searching. French President Francois Hollande, speaking Tuesday, said that it should be possible to identify all the victims using DNA analysis by the end of the week, sooner than authorities had previously suggested. In the meantime, the recovery of the victims\' personal belongings will start Wednesday, Menichini said. Among those personal belongings could be more cell phones belonging to the 144 passengers and six crew on board. Check out the latest from our correspondents . The details about Lubitz\'s correspondence with the flight school during his training were among several developments as investigators continued to delve into what caused the crash and Lubitz\'s possible motive for downing the jet. A Lufthansa spokesperson told CNN on Tuesday that Lubitz had a valid medical certificate, had passed all his examinations and "held all the licenses required." Earlier, a spokesman for the prosecutor\'s office in Dusseldorf, Christoph Kumpa, said medical records reveal Lubitz suffered from suicidal tendencies at some point before his aviation career and underwent psychotherapy before he got his pilot\'s license. Kumpa emphasized there\'s no evidence suggesting Lubitz was suicidal or acting aggressively before the crash. Investigators are looking into whether Lubitz feared his medical condition would cause him to lose his pilot\'s license, a European government official briefed on the investigation told CNN on Tuesday. While flying was "a big part of his life," the source said, it\'s only one theory being considered. Another source, a law enforcement official briefed on the investigation, also told CNN that authorities believe the primary motive for Lubitz to bring down the plane was that he feared he would not be allowed to fly because of his medical problems. Lubitz\'s girlfriend told investigators he had seen an eye doctor and a neuropsychologist, both of whom deemed him unfit to work recently and concluded he had psychological issues, the European government official said. But no matter what details emerge about his previous mental health struggles, there\'s more to the story, said Brian Russell, a forensic psychologist. "Psychology can explain why somebody would turn rage inward on themselves about the fact that maybe they weren\'t going to keep doing their job and they\'re upset about that and so they\'re suicidal," he said. "But there is no mental illness that explains why somebody then feels entitled to also take that rage and turn it outward on 149 other people who had nothing to do with the person\'s problems." Germanwings crash compensation: What we know . Who was the captain of Germanwings Flight 9525? CNN\'s Margot Haddad reported from Marseille and Pamela Brown from Dusseldorf, while Laura Smith-Spark wrote from London. CNN\'s Frederik Pleitgen, Pamela Boykoff, Antonia Mortensen, Sandrine Amiel and Anna-Maja Rappard contributed to this report.' # @noq - - SHORTER_ARTICLE = ' (CNN)The Palestinian Authority officially became the 123rd member of the International Criminal Court on Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian territories. The formal accession was marked with a ceremony at The Hague, in the Netherlands, where the court is based. The Palestinians signed the ICC\'s founding Rome Statute in January, when they also accepted its jurisdiction over alleged crimes committed "in the occupied Palestinian territory, including East Jerusalem, since June 13, 2014." Later that month, the ICC opened a preliminary examination into the situation in Palestinian territories, paving the way for possible war crimes investigations against Israelis. As members of the court, Palestinians may be subject to counter-charges as well. Israel and the United States, neither of which is an ICC member, opposed the Palestinians\' efforts to join the body. But Palestinian Foreign Minister Riad al-Malki, speaking at Wednesday\'s ceremony, said it was a move toward greater justice. "As Palestine formally becomes a State Party to the Rome Statute today, the world is also a step closer to ending a long era of impunity and injustice," he said, according to an ICC news release. "Indeed, today brings us closer to our shared goals of justice and peace." Judge Kuniko Ozaki, a vice president of the ICC, said acceding to the treaty was just the first step for the Palestinians. "As the Rome Statute today enters into force for the State of Palestine, Palestine acquires all the rights as well as responsibilities that come with being a State Party to the Statute. These are substantive commitments, which cannot be taken lightly," she said. Rights group Human Rights Watch welcomed the development. "Governments seeking to penalize Palestine for joining the ICC should immediately end their pressure, and countries that support universal acceptance of the court\'s treaty should speak out to welcome its membership," said Balkees Jarrah, international justice counsel for the group. "What\'s objectionable is the attempts to undermine international justice, not Palestine\'s decision to join a treaty to which over 100 countries around the world are members." In January, when the preliminary ICC examination was opened, Israeli Prime Minister Benjamin Netanyahu described it as an outrage, saying the court was overstepping its boundaries. The United States also said it "strongly" disagreed with the court\'s decision. "As we have said repeatedly, we do not believe that Palestine is a state and therefore we do not believe that it is eligible to join the ICC," the State Department said in a statement. It urged the warring sides to resolve their differences through direct negotiations. "We will continue to oppose actions against Israel at the ICC as counterproductive to the cause of peace," it said. But the ICC begs to differ with the definition of a state for its purposes and refers to the territories as "Palestine." While a preliminary examination is not a formal investigation, it allows the court to review evidence and determine whether to investigate suspects on both sides. Prosecutor Fatou Bensouda said her office would "conduct its analysis in full independence and impartiality." The war between Israel and Hamas militants in Gaza last summer left more than 2,000 people dead. The inquiry will include alleged war crimes committed since June. The International Criminal Court was set up in 2002 to prosecute genocide, crimes against humanity and war crimes. CNN\'s Vasco Cotovio, Kareem Khadder and Faith Karimi contributed to this report.' - - # The below article tests that we don't add any hypotheses outside of the top n_beams - IRAN_ARTICLE = " (CNN)The United States and its negotiating partners reached a very strong framework agreement with Iran in Lausanne, Switzerland, on Thursday that limits Iran's nuclear program in such a way as to effectively block it from building a nuclear weapon. Expect pushback anyway, if the recent past is any harbinger. Just last month, in an attempt to head off such an agreement, House Speaker John Boehner invited Israeli Prime Minister Benjamin Netanyahu to preemptively blast it before Congress, and 47 senators sent a letter to the Iranian leadership warning them away from a deal. The debate that has already begun since the announcement of the new framework will likely result in more heat than light. It will not be helped by the gathering swirl of dubious assumptions and doubtful assertions. Let us address some of these: . The most misleading assertion, despite universal rejection by experts, is that the negotiations' objective at the outset was the total elimination of any nuclear program in Iran. That is the position of Netanyahu and his acolytes in the U.S. Congress. But that is not and never was the objective. If it had been, there would have been no Iranian team at the negotiating table. Rather, the objective has always been to structure an agreement or series of agreements so that Iran could not covertly develop a nuclear arsenal before the United States and its allies could respond. The new framework has exceeded expectations in achieving that goal. It would reduce Iran's low-enriched uranium stockpile, cut by two-thirds its number of installed centrifuges and implement a rigorous inspection regime. Another dubious assumption of opponents is that the Iranian nuclear program is a covert weapons program. Despite sharp accusations by some in the United States and its allies, Iran denies having such a program, and U.S. intelligence contends that Iran has not yet made the decision to build a nuclear weapon. Iran's continued cooperation with International Atomic Energy Agency inspections is further evidence on this point, and we'll know even more about Iran's program in the coming months and years because of the deal. In fact, the inspections provisions that are part of this agreement are designed to protect against any covert action by the Iranians. What's more, the rhetoric of some members of Congress has implied that the negotiations have been between only the United States and Iran (i.e., the 47 senators' letter warning that a deal might be killed by Congress or a future president). This of course is not the case. The talks were between Iran and the five permanent members of the U.N. Security Council (United States, United Kingdom, France, China and Russia) plus Germany, dubbed the P5+1. While the United States has played a leading role in the effort, it negotiated the terms alongside its partners. If the agreement reached by the P5+1 is rejected by Congress, it could result in an unraveling of the sanctions on Iran and threaten NATO cohesion in other areas. Another questionable assertion is that this agreement contains a sunset clause, after which Iran will be free to do as it pleases. Again, this is not the case. Some of the restrictions on Iran's nuclear activities, such as uranium enrichment, will be eased or eliminated over time, as long as 15 years. But most importantly, the framework agreement includes Iran's ratification of the Additional Protocol, which allows IAEA inspectors expanded access to nuclear sites both declared and nondeclared. This provision will be permanent. It does not sunset. Thus, going forward, if Iran decides to enrich uranium to weapons-grade levels, monitors will be able to detect such a move in a matter of days and alert the U.N. Security Council. Many in Congress have said that the agreement should be a formal treaty requiring the Senate to \"advise and consent.\" But the issue is not suited for a treaty. Treaties impose equivalent obligations on all signatories. For example, the New START treaty limits Russia and the United States to 1,550 deployed strategic warheads. But any agreement with Iran will not be so balanced. The restrictions and obligations in the final framework agreement will be imposed almost exclusively on Iran. The P5+1 are obligated only to ease and eventually remove most but not all economic sanctions, which were imposed as leverage to gain this final deal. Finally some insist that any agreement must address Iranian missile programs, human rights violations or support for Hamas or Hezbollah. As important as these issues are, and they must indeed be addressed, they are unrelated to the most important aim of a nuclear deal: preventing a nuclear Iran. To include them in the negotiations would be a poison pill. This agreement should be judged on its merits and on how it affects the security of our negotiating partners and allies, including Israel. Those judgments should be fact-based, not based on questionable assertions or dubious assumptions." - - ARTICLE_SUBWAY = ' New York (CNN)When Liana Barrientos was 23 years old, she got married in Westchester County, New York. A year later, she got married again in Westchester County, but to a different man and without divorcing her first husband. Only 18 days after that marriage, she got hitched yet again. Then, Barrientos declared "I do" five more times, sometimes only within two weeks of each other. In 2010, she married once more, this time in the Bronx. In an application for a marriage license, she stated it was her "first and only" marriage. Barrientos, now 39, is facing two criminal counts of "offering a false instrument for filing in the first degree," referring to her false statements on the 2010 marriage license application, according to court documents. Prosecutors said the marriages were part of an immigration scam. On Friday, she pleaded not guilty at State Supreme Court in the Bronx, according to her attorney, Christopher Wright, who declined to comment further. After leaving court, Barrientos was arrested and charged with theft of service and criminal trespass for allegedly sneaking into the New York subway through an emergency exit, said Detective Annette Markowski, a police spokeswoman. In total, Barrientos has been married 10 times, with nine of her marriages occurring between 1999 and 2002. All occurred either in Westchester County, Long Island, New Jersey or the Bronx. She is believed to still be married to four men, and at one time, she was married to eight men at once, prosecutors say. Prosecutors said the immigration scam involved some of her husbands, who filed for permanent residence status shortly after the marriages. Any divorces happened only after such filings were approved. It was unclear whether any of the men will be prosecuted. The case was referred to the Bronx District Attorney\'s Office by Immigration and Customs Enforcement and the Department of Homeland Security\'s Investigation Division. Seven of the men are from so-called "red-flagged" countries, including Egypt, Turkey, Georgia, Pakistan and Mali. Her eighth husband, Rashid Rajput, was deported in 2006 to his native Pakistan after an investigation by the Joint Terrorism Task Force. If convicted, Barrientos faces up to four years in prison. Her next court appearance is scheduled for May 18.' - - dct = tok.batch_encode_plus( - [FRANCE_ARTICLE, SHORTER_ARTICLE, IRAN_ARTICLE, ARTICLE_SUBWAY], - max_length=1024, - padding="max_length", - truncation_strategy="only_first", - truncation=True, - return_tensors="pt", - ) - - self.assertEqual(1024, dct["input_ids"].shape[1]) - hypotheses_batch = hf.generate( - input_ids=dct["input_ids"].to(torch_device), - attention_mask=dct["attention_mask"].to(torch_device), - num_beams=2, - ) - assert hypotheses_batch[:, 1].eq(0).all().item() - - EXPECTED = [ - "A French prosecutor says he is not aware of any video footage from on board the plane. Two German " - "magazines claim to have found a cell phone video showing the crash. The publications say they watched " - "the video, which was found by a source close to the investigation. All 150 on board Germanwings Flight " - "9525 were killed.", - "Palestinian Authority becomes 123rd member of the International Criminal Court. The move gives the court " - "jurisdiction over alleged crimes in Palestinian territories. Israel and the United States opposed the " - "Palestinians' efforts to join the body. But Palestinian Foreign Minister Riad al-Malki said it was a " - "move toward greater justice.", - "U.S. and its negotiating partners reached a strong framework agreement with Iran. Peter Bergen: The " - "debate that has already begun will likely result in more heat than light. He says critics have made " - "dubious assumptions and doubtful assertions. Bergen says the goal was to block Iran from building a " - "nuclear weapon.", - "Liana Barrientos, 39, has been married 10 times, sometimes within two weeks of each other. Prosecutors " - "say the marriages were part of an immigration scam. She pleaded not guilty at State Supreme Court in the " - "Bronx on Friday. If convicted, she faces up to four years in prison.", - ] - - generated_summaries = tok.batch_decode( - hypotheses_batch.tolist(), clean_up_tokenization_spaces=True, skip_special_tokens=True - ) - assert generated_summaries == EXPECTED - class OPTStandaloneDecoderModelTester: def __init__( @@ -967,20 +815,22 @@ def test_retain_grad_hidden_states_attentions(self): return +@require_torch +@require_tokenizers class OPTEmbeddingsTest(unittest.TestCase): def setUp(self): super().setUp() self.path_model = "/home/younes/Desktop/Work/data/opt-350-m/" # TODO add the model on the hub - self.path_logits_meta = "/home/younes/Desktop/Work/metaseq-conversion/logits_metaseq_gpt2_tokenizer.p" # TODO add the logits somewhere? + # self.path_logits_meta = "/home/younes/Desktop/Work/metaseq-conversion/logits_metaseq_gpt2_tokenizer.p" # TODO add the logits somewhere? - @unittest.skip("Skipping unless we find a solution to store the model") + # @unittest.skip("Skipping unless we find a solution to store the model") def test_load_model(self): try: _ = OPTForCausalLM.from_pretrained(self.path_model) except BaseException: self.fail("Failed loading model") - @unittest.skip("Skipping unless we find a solution to store the logits") + @torch.no_grad() def test_logits(self): model = OPTForCausalLM.from_pretrained(self.path_model) model = model.eval() @@ -995,16 +845,25 @@ def test_logits(self): "Computers and mobile phones have taken", ] input_ids = tokenizer(prompts, return_tensors="pt", padding=True).input_ids - logits = model(input_ids)[0] - logits_meta = torch.load(self.path_logits_meta) - assert torch.allclose(logits, logits_meta) + logits = model(input_ids)[0].mean(dim=-1) + # logits_meta = torch.load(self.path_logits_meta) + logits_meta = torch.Tensor( + [ + [1.3851, -13.8923, -10.5229, -10.7533, -0.2309, -10.2384, -0.5365, -9.0947, -5.1670], + [-4.7073, -10.6276, -3.9415, -21.5242, -0.2822, -0.2822, -0.2822, -0.2822, -0.2822], + [0.6247, -3.4229, -8.9179, -1.4297, -14.1650, 1.4146, -9.0218, -0.2703, -0.2703], + [6.4783, -1.9913, -10.7926, -2.3336, 1.5092, -0.9974, -6.8213, 1.3477, 1.3477], + ] + ) + + assert torch.allclose(logits, logits_meta, atol=1e-4) class OPTGenerationTest(unittest.TestCase): def setUp(self): super().setUp() self.path_model = "/home/younes/Desktop/Work/data/opt-350-m/" # TODO add the model on the hub - self.path_logits_meta = "/home/younes/Desktop/Work/metaseq-conversion/logits_metaseq_gpt2_tokenizer.p" # TODO add the logits somewhere? + # self.path_logits_meta = "/home/younes/Desktop/Work/metaseq-conversion/logits_metaseq_gpt2_tokenizer.p" # TODO add the logits somewhere? @unittest.skip("Skipping unless we find a solution to store the model") def test_generation(self): From 38f7463a5ace3ca65a0dfd3b645e762614d9a071 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Fri, 6 May 2022 10:15:34 +0200 Subject: [PATCH 014/145] rm useless classes - remove opt for: - QA - cond generation - seq classif --- src/transformers/__init__.py | 6 ------ src/transformers/models/auto/modeling_auto.py | 6 ------ src/transformers/models/opt/__init__.py | 6 ------ src/transformers/utils/dummy_pt_objects.py | 21 ------------------- 4 files changed, 39 deletions(-) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index c016bb0902554..732377cee3ad2 100755 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -1275,9 +1275,6 @@ [ "OPT_PRETRAINED_MODEL_ARCHIVE_LIST", "OPTForCausalLM", - "OPTForConditionalGeneration", - "OPTForQuestionAnswering", - "OPTForSequenceClassification", "OPTModel", "OPTPretrainedModel", "PretrainedOPTModel", @@ -3546,9 +3543,6 @@ from .models.opt import ( OPT_PRETRAINED_MODEL_ARCHIVE_LIST, OPTForCausalLM, - OPTForConditionalGeneration, - OPTForQuestionAnswering, - OPTForSequenceClassification, OPTModel, OPTPretrainedModel, PretrainedOPTModel, diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index cfe1cb1b027b9..cf139f44fac47 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -144,7 +144,6 @@ ("xlm-roberta-xl", "XLMRobertaXLForMaskedLM"), ("xlm-roberta", "XLMRobertaForMaskedLM"), ("bart", "BartForConditionalGeneration"), - ("opt", "OPTForConditionalGeneration"), ("fsmt", "FSMTForConditionalGeneration"), ("longformer", "LongformerForMaskedLM"), ("roberta", "RobertaForMaskedLM"), @@ -205,7 +204,6 @@ ("marian", "MarianMTModel"), ("fsmt", "FSMTForConditionalGeneration"), ("bart", "BartForConditionalGeneration"), - ("opt", "OPTForConditionalGeneration"), ("longformer", "LongformerForMaskedLM"), ("roberta", "RobertaForMaskedLM"), ("data2vec-text", "Data2VecTextForMaskedLM"), @@ -364,7 +362,6 @@ ("distilbert", "DistilBertForMaskedLM"), ("albert", "AlbertForMaskedLM"), ("bart", "BartForConditionalGeneration"), - ("opt", "OPTForConditionalGeneration"), ("mbart", "MBartForConditionalGeneration"), ("camembert", "CamembertForMaskedLM"), ("xlm-roberta-xl", "XLMRobertaXLForMaskedLM"), @@ -413,7 +410,6 @@ ("mbart", "MBartForConditionalGeneration"), ("blenderbot", "BlenderbotForConditionalGeneration"), ("bart", "BartForConditionalGeneration"), - ("opt", "OPTForConditionalGeneration"), ("fsmt", "FSMTForConditionalGeneration"), ("encoder-decoder", "EncoderDecoderModel"), ("xlm-prophetnet", "XLMProphetNetForConditionalGeneration"), @@ -454,7 +450,6 @@ ("xlm-roberta", "XLMRobertaForSequenceClassification"), ("mbart", "MBartForSequenceClassification"), ("bart", "BartForSequenceClassification"), - ("opt", "OPTForSequenceClassification"), ("longformer", "LongformerForSequenceClassification"), ("roberta", "RobertaForSequenceClassification"), ("data2vec-text", "Data2VecTextForSequenceClassification"), @@ -502,7 +497,6 @@ ("albert", "AlbertForQuestionAnswering"), ("camembert", "CamembertForQuestionAnswering"), ("bart", "BartForQuestionAnswering"), - ("opt", "OPTForQuestionAnswering"), ("mbart", "MBartForQuestionAnswering"), ("longformer", "LongformerForQuestionAnswering"), ("xlm-roberta-xl", "XLMRobertaXLForQuestionAnswering"), diff --git a/src/transformers/models/opt/__init__.py b/src/transformers/models/opt/__init__.py index 641764a29b341..422059b7da18a 100644 --- a/src/transformers/models/opt/__init__.py +++ b/src/transformers/models/opt/__init__.py @@ -32,9 +32,6 @@ _import_structure["modeling_opt"] = [ "OPT_PRETRAINED_MODEL_ARCHIVE_LIST", "OPTForCausalLM", - "OPTForConditionalGeneration", - "OPTForQuestionAnswering", - "OPTForSequenceClassification", "OPTModel", "OPTPretrainedModel", "PretrainedOPTModel", @@ -65,9 +62,6 @@ from .modeling_opt import ( OPT_PRETRAINED_MODEL_ARCHIVE_LIST, OPTForCausalLM, - OPTForConditionalGeneration, - OPTForQuestionAnswering, - OPTForSequenceClassification, OPTModel, OPTPretrainedModel, PretrainedOPTModel, diff --git a/src/transformers/utils/dummy_pt_objects.py b/src/transformers/utils/dummy_pt_objects.py index 5d0c395c85116..cbe4364466d85 100644 --- a/src/transformers/utils/dummy_pt_objects.py +++ b/src/transformers/utils/dummy_pt_objects.py @@ -2948,27 +2948,6 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["torch"]) -class OPTForConditionalGeneration(metaclass=DummyObject): - _backends = ["torch"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["torch"]) - - -class OPTForQuestionAnswering(metaclass=DummyObject): - _backends = ["torch"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["torch"]) - - -class OPTForSequenceClassification(metaclass=DummyObject): - _backends = ["torch"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["torch"]) - - class OPTModel(metaclass=DummyObject): _backends = ["torch"] From c6f3a69b5be91d5024ca90728362c72ec4afb889 Mon Sep 17 00:00:00 2001 From: Arthur <48595927+ArthurZucker@users.noreply.github.com> Date: Fri, 6 May 2022 10:27:24 +0200 Subject: [PATCH 015/145] Removed autodoc calls to non-existant classes TOkenizers are not implemented --- docs/source/en/model_doc/opt.mdx | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/docs/source/en/model_doc/opt.mdx b/docs/source/en/model_doc/opt.mdx index e1a37320cd719..63e478cedbaae 100644 --- a/docs/source/en/model_doc/opt.mdx +++ b/docs/source/en/model_doc/opt.mdx @@ -34,15 +34,6 @@ The original code can be found [here](). [[autodoc]] OPTConfig - all -## OPTTokenizer - -[[autodoc]] OPTTokenizer - - all - -## OPTTokenizerFast - -[[autodoc]] OPTTokenizerFast - - all ## OPTModel @@ -125,4 +116,4 @@ The original code can be found [here](). ## FlaxOPTForCausalLM [[autodoc]] FlaxOPTForCausalLM - - __call__ \ No newline at end of file + - __call__ From 30d3db297d65ed687e3fe4f578d932b2c9198383 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Fri, 6 May 2022 11:33:46 +0200 Subject: [PATCH 016/145] Update src/transformers/__init__.py Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com> --- src/transformers/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 732377cee3ad2..dbcfa7f85bb5f 100755 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -484,7 +484,6 @@ _import_structure["models.mpnet"].append("MPNetTokenizerFast") _import_structure["models.mt5"].append("MT5TokenizerFast") _import_structure["models.openai"].append("OpenAIGPTTokenizerFast") - _import_structure["models.opt"].append("OPTTokenizerFast") _import_structure["models.pegasus"].append("PegasusTokenizerFast") _import_structure["models.realm"].append("RealmTokenizerFast") _import_structure["models.reformer"].append("ReformerTokenizerFast") From f903445dbbcb00e6627648875038a89efc92ffb2 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Fri, 6 May 2022 11:33:54 +0200 Subject: [PATCH 017/145] Update src/transformers/__init__.py Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com> --- src/transformers/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index dbcfa7f85bb5f..c2d89fe289fe2 100755 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -2893,7 +2893,6 @@ from .models.mpnet import MPNetTokenizerFast from .models.mt5 import MT5TokenizerFast from .models.openai import OpenAIGPTTokenizerFast - from .models.opt import OPTTokenizerFast from .models.pegasus import PegasusTokenizerFast from .models.realm import RealmTokenizerFast from .models.reformer import ReformerTokenizerFast From bb4ab4ae963b7961d11b730cb31210e161d28e56 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Fri, 6 May 2022 11:34:00 +0200 Subject: [PATCH 018/145] Update src/transformers/models/auto/modeling_tf_auto.py Co-authored-by: Arthur <48595927+ArthurZucker@users.noreply.github.com> --- src/transformers/models/auto/modeling_tf_auto.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/transformers/models/auto/modeling_tf_auto.py b/src/transformers/models/auto/modeling_tf_auto.py index d5e74dd3877a5..334900a0d2ee6 100644 --- a/src/transformers/models/auto/modeling_tf_auto.py +++ b/src/transformers/models/auto/modeling_tf_auto.py @@ -120,7 +120,6 @@ ("albert", "TFAlbertForMaskedLM"), ("marian", "TFMarianMTModel"), ("bart", "TFBartForConditionalGeneration"), - ("opt", "TFOPTForConditionalGeneration"), ("camembert", "TFCamembertForMaskedLM"), ("xlm-roberta", "TFXLMRobertaForMaskedLM"), ("longformer", "TFLongformerForMaskedLM"), From 2a6e2884d0427237ebfbed5e0d0359ffbc07233d Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Fri, 6 May 2022 11:53:52 +0200 Subject: [PATCH 019/145] Replaced OPTTokeniser with GPT2 tokenizer --- tests/models/opt/test_modeling_flax_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/models/opt/test_modeling_flax_opt.py b/tests/models/opt/test_modeling_flax_opt.py index 10e8fdf6c57f3..cdea3cea638f7 100644 --- a/tests/models/opt/test_modeling_flax_opt.py +++ b/tests/models/opt/test_modeling_flax_opt.py @@ -16,7 +16,7 @@ import numpy as np import timeout_decorator # noqa -from transformers import OPTConfig, OPTTokenizer, is_flax_available +from transformers import OPTConfig, GPT2Tokenizer, is_flax_available from transformers.testing_utils import require_flax, slow from ...generation.test_generation_flax_utils import FlaxGenerationTesterMixin @@ -434,7 +434,7 @@ def test_summarization_fast(self): @slow def test_cnn_summarization_same_as_fairseq(self): model = FlaxOPTForConditionalGeneration.from_pretrained("facebook/opt-large-cnn") - tokenizer = OPTTokenizer.from_pretrained("facebook/opt-large-cnn") + tokenizer = GPT2Tokenizer.from_pretrained("patrickvonplaten/opt_gpt2_tokenizer") FRANCE_ARTICLE = ' Marseille, France (CNN)The French prosecutor leading an investigation into the crash of Germanwings Flight 9525 insisted Wednesday that he was not aware of any video footage from on board the plane. Marseille prosecutor Brice Robin told CNN that "so far no videos were used in the crash investigation." He added, "A person who has such a video needs to immediately give it to the investigators." Robin\'s comments follow claims by two magazines, German daily Bild and French Paris Match, of a cell phone video showing the harrowing final seconds from on board Germanwings Flight 9525 as it crashed into the French Alps. All 150 on board were killed. Paris Match and Bild reported that the video was recovered from a phone at the wreckage site. The two publications described the supposed video, but did not post it on their websites. The publications said that they watched the video, which was found by a source close to the investigation. "One can hear cries of \'My God\' in several languages," Paris Match reported. "Metallic banging can also be heard more than three times, perhaps of the pilot trying to open the cockpit door with a heavy object. Towards the end, after a heavy shake, stronger than the others, the screaming intensifies. Then nothing." "It is a very disturbing scene," said Julian Reichelt, editor-in-chief of Bild online. An official with France\'s accident investigation agency, the BEA, said the agency is not aware of any such video. Lt. Col. Jean-Marc Menichini, a French Gendarmerie spokesman in charge of communications on rescue efforts around the Germanwings crash site, told CNN that the reports were "completely wrong" and "unwarranted." Cell phones have been collected at the site, he said, but that they "hadn\'t been exploited yet." Menichini said he believed the cell phones would need to be sent to the Criminal Research Institute in Rosny sous-Bois, near Paris, in order to be analyzed by specialized technicians working hand-in-hand with investigators. But none of the cell phones found so far have been sent to the institute, Menichini said. Asked whether staff involved in the search could have leaked a memory card to the media, Menichini answered with a categorical "no." Reichelt told "Erin Burnett: Outfront" that he had watched the video and stood by the report, saying Bild and Paris Match are "very confident" that the clip is real. He noted that investigators only revealed they\'d recovered cell phones from the crash site after Bild and Paris Match published their reports. "That is something we did not know before. ... Overall we can say many things of the investigation weren\'t revealed by the investigation at the beginning," he said. What was mental state of Germanwings co-pilot? German airline Lufthansa confirmed Tuesday that co-pilot Andreas Lubitz had battled depression years before he took the controls of Germanwings Flight 9525, which he\'s accused of deliberately crashing last week in the French Alps. Lubitz told his Lufthansa flight training school in 2009 that he had a "previous episode of severe depression," the airline said Tuesday. Email correspondence between Lubitz and the school discovered in an internal investigation, Lufthansa said, included medical documents he submitted in connection with resuming his flight training. The announcement indicates that Lufthansa, the parent company of Germanwings, knew of Lubitz\'s battle with depression, allowed him to continue training and ultimately put him in the cockpit. Lufthansa, whose CEO Carsten Spohr previously said Lubitz was 100% fit to fly, described its statement Tuesday as a "swift and seamless clarification" and said it was sharing the information and documents -- including training and medical records -- with public prosecutors. Spohr traveled to the crash site Wednesday, where recovery teams have been working for the past week to recover human remains and plane debris scattered across a steep mountainside. He saw the crisis center set up in Seyne-les-Alpes, laid a wreath in the village of Le Vernet, closer to the crash site, where grieving families have left flowers at a simple stone memorial. Menichini told CNN late Tuesday that no visible human remains were left at the site but recovery teams would keep searching. French President Francois Hollande, speaking Tuesday, said that it should be possible to identify all the victims using DNA analysis by the end of the week, sooner than authorities had previously suggested. In the meantime, the recovery of the victims\' personal belongings will start Wednesday, Menichini said. Among those personal belongings could be more cell phones belonging to the 144 passengers and six crew on board. Check out the latest from our correspondents . The details about Lubitz\'s correspondence with the flight school during his training were among several developments as investigators continued to delve into what caused the crash and Lubitz\'s possible motive for downing the jet. A Lufthansa spokesperson told CNN on Tuesday that Lubitz had a valid medical certificate, had passed all his examinations and "held all the licenses required." Earlier, a spokesman for the prosecutor\'s office in Dusseldorf, Christoph Kumpa, said medical records reveal Lubitz suffered from suicidal tendencies at some point before his aviation career and underwent psychotherapy before he got his pilot\'s license. Kumpa emphasized there\'s no evidence suggesting Lubitz was suicidal or acting aggressively before the crash. Investigators are looking into whether Lubitz feared his medical condition would cause him to lose his pilot\'s license, a European government official briefed on the investigation told CNN on Tuesday. While flying was "a big part of his life," the source said, it\'s only one theory being considered. Another source, a law enforcement official briefed on the investigation, also told CNN that authorities believe the primary motive for Lubitz to bring down the plane was that he feared he would not be allowed to fly because of his medical problems. Lubitz\'s girlfriend told investigators he had seen an eye doctor and a neuropsychologist, both of whom deemed him unfit to work recently and concluded he had psychological issues, the European government official said. But no matter what details emerge about his previous mental health struggles, there\'s more to the story, said Brian Russell, a forensic psychologist. "Psychology can explain why somebody would turn rage inward on themselves about the fact that maybe they weren\'t going to keep doing their job and they\'re upset about that and so they\'re suicidal," he said. "But there is no mental illness that explains why somebody then feels entitled to also take that rage and turn it outward on 149 other people who had nothing to do with the person\'s problems." Germanwings crash compensation: What we know . Who was the captain of Germanwings Flight 9525? CNN\'s Margot Haddad reported from Marseille and Pamela Brown from Dusseldorf, while Laura Smith-Spark wrote from London. CNN\'s Frederik Pleitgen, Pamela Boykoff, Antonia Mortensen, Sandrine Amiel and Anna-Maja Rappard contributed to this report.' # @noq From cb853fd5c2d1000a4814a14a0389df9b8526545b Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Fri, 6 May 2022 11:54:39 +0200 Subject: [PATCH 020/145] added GPT2Tokenizer.from_pretrained("patrickvonplaten/opt_gpt2_tokenizer") --- tests/models/opt/test_modeling_flax_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/models/opt/test_modeling_flax_opt.py b/tests/models/opt/test_modeling_flax_opt.py index cdea3cea638f7..3037db5cb459c 100644 --- a/tests/models/opt/test_modeling_flax_opt.py +++ b/tests/models/opt/test_modeling_flax_opt.py @@ -418,7 +418,7 @@ def test_model_from_pretrained(self): @slow def test_summarization_fast(self): model = FlaxOPTForConditionalGeneration.from_pretrained("sshleifer/distilopt-cnn-6-6") - tokenizer = OPTTokenizer.from_pretrained("sshleifer/distilopt-cnn-6-6") + tokenizer = GPT2Tokenizer.from_pretrained("patrickvonplaten/opt_gpt2_tokenizer") input_str = "This sentence is made of three parts. Each part is important on its own. One part is about animals, the other part about planes, and the last part about housing." From 337e71fcb183bfc7cdfbcbb4d9c53b532bf122a7 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Fri, 6 May 2022 12:08:29 +0200 Subject: [PATCH 021/145] Removed OPTTokenizer --- src/transformers/__init__.py | 4 ++-- src/transformers/models/opt/__init__.py | 7 +------ src/transformers/models/opt/modeling_flax_opt.py | 2 +- src/transformers/models/opt/modeling_tf_opt.py | 2 +- tests/models/opt/test_modeling_tf_opt.py | 6 +++--- 5 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index c2d89fe289fe2..89d91fd65d8ee 100755 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -238,7 +238,7 @@ "NystromformerConfig", ], "models.openai": ["OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP", "OpenAIGPTConfig", "OpenAIGPTTokenizer"], - "models.opt": ["OPTConfig", "OPTTokenizer"], + "models.opt": ["OPTConfig"], "models.pegasus": ["PEGASUS_PRETRAINED_CONFIG_ARCHIVE_MAP", "PegasusConfig", "PegasusTokenizer"], "models.perceiver": ["PERCEIVER_PRETRAINED_CONFIG_ARCHIVE_MAP", "PerceiverConfig", "PerceiverTokenizer"], "models.phobert": ["PhobertTokenizer"], @@ -2668,7 +2668,7 @@ from .models.mt5 import MT5Config from .models.nystromformer import NYSTROMFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, NystromformerConfig from .models.openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig, OpenAIGPTTokenizer - from .models.opt import OPTConfig, OPTTokenizer + from .models.opt import OPTConfig from .models.pegasus import PEGASUS_PRETRAINED_CONFIG_ARCHIVE_MAP, PegasusConfig, PegasusTokenizer from .models.perceiver import PERCEIVER_PRETRAINED_CONFIG_ARCHIVE_MAP, PerceiverConfig, PerceiverTokenizer from .models.phobert import PhobertTokenizer diff --git a/src/transformers/models/opt/__init__.py b/src/transformers/models/opt/__init__.py index 422059b7da18a..1f299c1d4aae3 100644 --- a/src/transformers/models/opt/__init__.py +++ b/src/transformers/models/opt/__init__.py @@ -22,11 +22,8 @@ _import_structure = { "configuration_opt": ["OPT_PRETRAINED_CONFIG_ARCHIVE_MAP", "OPTConfig", "OPTOnnxConfig"], - "tokenization_opt": ["OPTTokenizer"], } -if is_tokenizers_available(): - _import_structure["tokenization_opt_fast"] = ["OPTTokenizerFast"] if is_torch_available(): _import_structure["modeling_opt"] = [ @@ -53,10 +50,8 @@ if TYPE_CHECKING: from .configuration_opt import OPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OPTConfig, OPTOnnxConfig - from .tokenization_opt import OPTTokenizer - if is_tokenizers_available(): - from .tokenization_opt_fast import OPTTokenizerFast + if is_torch_available(): from .modeling_opt import ( diff --git a/src/transformers/models/opt/modeling_flax_opt.py b/src/transformers/models/opt/modeling_flax_opt.py index 3662620b0fd0f..f58775a5f42a6 100644 --- a/src/transformers/models/opt/modeling_flax_opt.py +++ b/src/transformers/models/opt/modeling_flax_opt.py @@ -55,7 +55,7 @@ _CHECKPOINT_FOR_DOC = "" _CONFIG_FOR_DOC = "OPTConfig" -_TOKENIZER_FOR_DOC = "OPTTokenizer" +_TOKENIZER_FOR_DOC = "GPT2Tokenizer" OPT_START_DOCSTRING = r""" diff --git a/src/transformers/models/opt/modeling_tf_opt.py b/src/transformers/models/opt/modeling_tf_opt.py index d5f733a9f11d6..d668f5a96abc2 100644 --- a/src/transformers/models/opt/modeling_tf_opt.py +++ b/src/transformers/models/opt/modeling_tf_opt.py @@ -56,7 +56,7 @@ _CHECKPOINT_FOR_DOC = "facebook/bart-large" _CONFIG_FOR_DOC = "OPTConfig" -_TOKENIZER_FOR_DOC = "OPTTokenizer" +_TOKENIZER_FOR_DOC = "GPT2Tokenizer" LARGE_NEGATIVE = -1e8 diff --git a/tests/models/opt/test_modeling_tf_opt.py b/tests/models/opt/test_modeling_tf_opt.py index 9d3bb3f7107cd..645be906f771d 100644 --- a/tests/models/opt/test_modeling_tf_opt.py +++ b/tests/models/opt/test_modeling_tf_opt.py @@ -17,7 +17,7 @@ import numpy as np -from transformers import OPTConfig, OPTTokenizer, is_tf_available +from transformers import OPTConfig, GPT2Tokenizer, is_tf_available from transformers.testing_utils import require_tf, slow from transformers.utils import cached_property @@ -414,7 +414,7 @@ def test_cnn_summarization_same_as_fairseq_hard(self): @cached_property def tok(self): - return OPTTokenizer.from_pretrained("facebook/opt-large") + return GPT2Tokenizer.from_pretrained("patrickvonplaten/opt_gpt2_tokenizer") @slow @@ -424,7 +424,7 @@ class FasterTFOPTModelIntegrationTests(unittest.TestCase): @cached_property def tok(self): - return OPTTokenizer.from_pretrained("facebook/opt-large") + return GPT2Tokenizer.from_pretrained("patrickvonplaten/opt_gpt2_tokenizer") @cached_property def xsum_1_1_model(self): From 0d9130f178c4a81f829f8864ab8d2d8e691964d5 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Fri, 6 May 2022 12:13:52 +0200 Subject: [PATCH 022/145] make style --- src/transformers/models/opt/__init__.py | 2 -- tests/models/opt/test_modeling_flax_opt.py | 2 +- tests/models/opt/test_modeling_tf_opt.py | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/opt/__init__.py b/src/transformers/models/opt/__init__.py index 1f299c1d4aae3..eef337d3bbd93 100644 --- a/src/transformers/models/opt/__init__.py +++ b/src/transformers/models/opt/__init__.py @@ -51,8 +51,6 @@ if TYPE_CHECKING: from .configuration_opt import OPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OPTConfig, OPTOnnxConfig - - if is_torch_available(): from .modeling_opt import ( OPT_PRETRAINED_MODEL_ARCHIVE_LIST, diff --git a/tests/models/opt/test_modeling_flax_opt.py b/tests/models/opt/test_modeling_flax_opt.py index 3037db5cb459c..665ca8db79122 100644 --- a/tests/models/opt/test_modeling_flax_opt.py +++ b/tests/models/opt/test_modeling_flax_opt.py @@ -16,7 +16,7 @@ import numpy as np import timeout_decorator # noqa -from transformers import OPTConfig, GPT2Tokenizer, is_flax_available +from transformers import GPT2Tokenizer, OPTConfig, is_flax_available from transformers.testing_utils import require_flax, slow from ...generation.test_generation_flax_utils import FlaxGenerationTesterMixin diff --git a/tests/models/opt/test_modeling_tf_opt.py b/tests/models/opt/test_modeling_tf_opt.py index 645be906f771d..b55506e6152bb 100644 --- a/tests/models/opt/test_modeling_tf_opt.py +++ b/tests/models/opt/test_modeling_tf_opt.py @@ -17,7 +17,7 @@ import numpy as np -from transformers import OPTConfig, GPT2Tokenizer, is_tf_available +from transformers import GPT2Tokenizer, OPTConfig, is_tf_available from transformers.testing_utils import require_tf, slow from transformers.utils import cached_property From 290b7f0cd5b19c2e47999cacdb3f2dda4746ec0f Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Fri, 6 May 2022 12:16:57 +0200 Subject: [PATCH 023/145] Make style replaces ``` ...).unsqueeze(``` by ``` >>>).unsqueeze(``` --- docs/source/en/model_doc/bert-generation.mdx | 4 ++-- docs/source/en/model_doc/luke.mdx | 2 +- .../modeling_encoder_decoder.py | 2 +- .../modeling_tf_encoder_decoder.py | 2 +- .../models/gpt2/modeling_tf_gpt2.py | 2 +- .../models/imagegpt/modeling_imagegpt.py | 2 +- .../models/longformer/modeling_longformer.py | 8 +++---- src/transformers/models/luke/modeling_luke.py | 6 ++--- .../models/openai/modeling_openai.py | 2 +- .../models/openai/modeling_tf_openai.py | 4 ++-- .../models/prophetnet/modeling_prophetnet.py | 6 ++--- src/transformers/models/rag/modeling_rag.py | 4 ++-- src/transformers/models/rag/retrieval_rag.py | 2 +- .../models/realm/modeling_realm.py | 2 +- .../modeling_tf_speech_to_text.py | 2 +- .../models/t5/modeling_flax_t5.py | 2 +- src/transformers/models/t5/modeling_t5.py | 6 ++--- src/transformers/models/t5/modeling_tf_t5.py | 6 ++--- .../models/tapas/modeling_tapas.py | 2 +- .../models/tapas/modeling_tf_tapas.py | 2 +- .../modeling_tf_vision_encoder_decoder.py | 2 +- .../models/wav2vec2/modeling_flax_wav2vec2.py | 4 ++-- src/transformers/models/xlm/modeling_xlm.py | 2 +- .../xlm_prophetnet/modeling_xlm_prophetnet.py | 4 ++-- .../models/xlnet/modeling_tf_xlnet.py | 8 +++---- .../models/xlnet/modeling_xlnet.py | 24 +++++++++---------- 26 files changed, 56 insertions(+), 56 deletions(-) diff --git a/docs/source/en/model_doc/bert-generation.mdx b/docs/source/en/model_doc/bert-generation.mdx index e300917ea5e6f..3c6c229b6a593 100644 --- a/docs/source/en/model_doc/bert-generation.mdx +++ b/docs/source/en/model_doc/bert-generation.mdx @@ -49,7 +49,7 @@ Usage: >>> input_ids = tokenizer( ... "This is a long article to summarize", add_special_tokens=False, return_tensors="pt" -... ).input_ids +>>> ).input_ids >>> labels = tokenizer("This is a short summary", return_tensors="pt").input_ids >>> # train... @@ -67,7 +67,7 @@ Usage: >>> input_ids = tokenizer( ... "This is the first sentence. This is the second sentence.", add_special_tokens=False, return_tensors="pt" -... ).input_ids +>>> ).input_ids >>> outputs = sentence_fuser.generate(input_ids) diff --git a/docs/source/en/model_doc/luke.mdx b/docs/source/en/model_doc/luke.mdx index 6900367bb8369..1bbf6acb84544 100644 --- a/docs/source/en/model_doc/luke.mdx +++ b/docs/source/en/model_doc/luke.mdx @@ -97,7 +97,7 @@ Example: >>> entities = [ ... "Beyoncé", ... "Los Angeles", -... ] # Wikipedia entity titles corresponding to the entity mentions "Beyoncé" and "Los Angeles" +>>> ] # Wikipedia entity titles corresponding to the entity mentions "Beyoncé" and "Los Angeles" >>> entity_spans = [(0, 7), (17, 28)] # character-based entity spans corresponding to "Beyoncé" and "Los Angeles" >>> inputs = tokenizer(text, entities=entities, entity_spans=entity_spans, add_prefix_space=True, return_tensors="pt") >>> outputs = model(**inputs) diff --git a/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py b/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py index 6eae4c876f442..972b80db7b4db 100644 --- a/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py +++ b/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py @@ -457,7 +457,7 @@ def forward( >>> tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") >>> model = EncoderDecoderModel.from_encoder_decoder_pretrained( ... "bert-base-uncased", "bert-base-uncased" - ... ) # initialize Bert2Bert from pre-trained checkpoints + >>> ) # initialize Bert2Bert from pre-trained checkpoints >>> # training >>> model.config.decoder_start_token_id = tokenizer.cls_token_id diff --git a/src/transformers/models/encoder_decoder/modeling_tf_encoder_decoder.py b/src/transformers/models/encoder_decoder/modeling_tf_encoder_decoder.py index 96c93d31cac25..9e92e767b1b85 100644 --- a/src/transformers/models/encoder_decoder/modeling_tf_encoder_decoder.py +++ b/src/transformers/models/encoder_decoder/modeling_tf_encoder_decoder.py @@ -528,7 +528,7 @@ def call( >>> # forward >>> input_ids = tokenizer.encode( ... "Hello, my dog is cute", add_special_tokens=True, return_tensors="tf" - ... ) # Batch size 1 + >>> ) # Batch size 1 >>> outputs = model(input_ids=input_ids, decoder_input_ids=input_ids) >>> # training diff --git a/src/transformers/models/gpt2/modeling_tf_gpt2.py b/src/transformers/models/gpt2/modeling_tf_gpt2.py index 45d29b6779ee1..3a11f46bdb3be 100644 --- a/src/transformers/models/gpt2/modeling_tf_gpt2.py +++ b/src/transformers/models/gpt2/modeling_tf_gpt2.py @@ -1061,7 +1061,7 @@ def call( >>> embedding_layer = model.resize_token_embeddings( ... len(tokenizer) - ... ) # Update the model embeddings with the new vocabulary size + >>> ) # Update the model embeddings with the new vocabulary size >>> choices = ["Hello, my dog is cute [CLS]", "Hello, my cat is cute [CLS]"] >>> encoded_choices = [tokenizer.encode(s) for s in choices] diff --git a/src/transformers/models/imagegpt/modeling_imagegpt.py b/src/transformers/models/imagegpt/modeling_imagegpt.py index 22186a6159e37..5866744bd8d0a 100755 --- a/src/transformers/models/imagegpt/modeling_imagegpt.py +++ b/src/transformers/models/imagegpt/modeling_imagegpt.py @@ -1000,7 +1000,7 @@ def forward( >>> samples = output[:, 1:].cpu().detach().numpy() >>> samples_img = [ ... np.reshape(np.rint(127.5 * (clusters[s] + 1.0)), [n_px, n_px, 3]).astype(np.uint8) for s in samples - ... ] # convert color cluster tokens back to pixels + >>> ] # convert color cluster tokens back to pixels >>> f, axes = plt.subplots(1, batch_size, dpi=300) >>> for img, ax in zip(samples_img, axes): diff --git a/src/transformers/models/longformer/modeling_longformer.py b/src/transformers/models/longformer/modeling_longformer.py index 20a2e9d239e24..32210ac65b60b 100755 --- a/src/transformers/models/longformer/modeling_longformer.py +++ b/src/transformers/models/longformer/modeling_longformer.py @@ -1634,10 +1634,10 @@ def forward( >>> attention_mask = torch.ones( ... input_ids.shape, dtype=torch.long, device=input_ids.device - ... ) # initialize to local attention + >>> ) # initialize to local attention >>> global_attention_mask = torch.zeros( ... input_ids.shape, dtype=torch.long, device=input_ids.device - ... ) # initialize to global attention to be deactivated for all tokens + >>> ) # initialize to global attention to be deactivated for all tokens >>> global_attention_mask[ ... :, ... [ @@ -1645,7 +1645,7 @@ def forward( ... 4, ... 21, ... ], - ... ] = 1 # Set global attention to random tokens for the sake of this example + >>> ] = 1 # Set global attention to random tokens for the sake of this example >>> # Usually, set global attention based on the task. For example, >>> # classification: the token >>> # QA: question tokens @@ -2025,7 +2025,7 @@ def forward( >>> answer_tokens = all_tokens[torch.argmax(start_logits) : torch.argmax(end_logits) + 1] >>> answer = tokenizer.decode( ... tokenizer.convert_tokens_to_ids(answer_tokens) - ... ) # remove space prepending space token + >>> ) # remove space prepending space token ```""" return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/luke/modeling_luke.py b/src/transformers/models/luke/modeling_luke.py index 7388e2031bab3..cd5a53ddae305 100644 --- a/src/transformers/models/luke/modeling_luke.py +++ b/src/transformers/models/luke/modeling_luke.py @@ -953,11 +953,11 @@ def forward( >>> entities = [ ... "Beyoncé", ... "Los Angeles", - ... ] # Wikipedia entity titles corresponding to the entity mentions "Beyoncé" and "Los Angeles" + >>> ] # Wikipedia entity titles corresponding to the entity mentions "Beyoncé" and "Los Angeles" >>> entity_spans = [ ... (0, 7), ... (17, 28), - ... ] # character-based entity spans corresponding to "Beyoncé" and "Los Angeles" + >>> ] # character-based entity spans corresponding to "Beyoncé" and "Los Angeles" >>> encoding = tokenizer( ... text, entities=entities, entity_spans=entity_spans, add_prefix_space=True, return_tensors="pt" @@ -1435,7 +1435,7 @@ def forward( >>> entity_spans = [ ... (0, 7), ... (17, 28), - ... ] # character-based entity spans corresponding to "Beyoncé" and "Los Angeles" + >>> ] # character-based entity spans corresponding to "Beyoncé" and "Los Angeles" >>> inputs = tokenizer(text, entity_spans=entity_spans, return_tensors="pt") >>> outputs = model(**inputs) >>> logits = outputs.logits diff --git a/src/transformers/models/openai/modeling_openai.py b/src/transformers/models/openai/modeling_openai.py index 2262db9aa8cf8..8ded535cef102 100644 --- a/src/transformers/models/openai/modeling_openai.py +++ b/src/transformers/models/openai/modeling_openai.py @@ -674,7 +674,7 @@ def forward( >>> model = OpenAIGPTDoubleHeadsModel.from_pretrained("openai-gpt") >>> tokenizer.add_special_tokens( ... {"cls_token": "[CLS]"} - ... ) # Add a [CLS] to the vocabulary (we should train it also!) + >>> ) # Add a [CLS] to the vocabulary (we should train it also!) >>> model.resize_token_embeddings(len(tokenizer)) >>> choices = ["Hello, my dog is cute [CLS]", "Hello, my cat is cute [CLS]"] diff --git a/src/transformers/models/openai/modeling_tf_openai.py b/src/transformers/models/openai/modeling_tf_openai.py index 24a7935eb0050..5215ad7c2ff6e 100644 --- a/src/transformers/models/openai/modeling_tf_openai.py +++ b/src/transformers/models/openai/modeling_tf_openai.py @@ -693,9 +693,9 @@ def call( >>> inputs = {k: tf.expand_dims(v, 0) for k, v in encoding.items()} >>> inputs["mc_token_ids"] = tf.constant( ... [inputs["input_ids"].shape[-1] - 1, inputs["input_ids"].shape[-1] - 1] - ... )[ + >>> )[ ... None, : - ... ] # Batch size 1 + >>> ] # Batch size 1 >>> outputs = model(inputs) >>> lm_prediction_scores, mc_prediction_scores = outputs[:2] ```""" diff --git a/src/transformers/models/prophetnet/modeling_prophetnet.py b/src/transformers/models/prophetnet/modeling_prophetnet.py index c869d6373bc5a..84fb75f3f3340 100644 --- a/src/transformers/models/prophetnet/modeling_prophetnet.py +++ b/src/transformers/models/prophetnet/modeling_prophetnet.py @@ -1813,7 +1813,7 @@ def forward( >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="pt" - ... ).input_ids # Batch size 1 + >>> ).input_ids # Batch size 1 >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids # Batch size 1 >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids) @@ -1935,7 +1935,7 @@ def forward( >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="pt" - ... ).input_ids # Batch size 1 + >>> ).input_ids # Batch size 1 >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids # Batch size 1 >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids) @@ -2202,7 +2202,7 @@ def forward( >>> input_ids = tokenizer_enc(ARTICLE, return_tensors="pt").input_ids >>> labels = tokenizer_dec( ... "us rejects charges against its ambassador in bolivia", return_tensors="pt" - ... ).input_ids + >>> ).input_ids >>> outputs = model(input_ids=input_ids, decoder_input_ids=labels[:, :-1], labels=labels[:, 1:]) >>> loss = outputs.loss diff --git a/src/transformers/models/rag/modeling_rag.py b/src/transformers/models/rag/modeling_rag.py index 205e825cbc1fa..642b13c580c06 100644 --- a/src/transformers/models/rag/modeling_rag.py +++ b/src/transformers/models/rag/modeling_rag.py @@ -826,7 +826,7 @@ def forward( >>> docs_dict = retriever(input_ids.numpy(), question_hidden_states.detach().numpy(), return_tensors="pt") >>> doc_scores = torch.bmm( ... question_hidden_states.unsqueeze(1), docs_dict["retrieved_doc_embeds"].float().transpose(1, 2) - ... ).squeeze(1) + >>> ).squeeze(1) >>> # 3. Forward to generator >>> outputs = model( ... context_input_ids=docs_dict["context_input_ids"], @@ -1293,7 +1293,7 @@ def forward( >>> docs_dict = retriever(input_ids.numpy(), question_hidden_states.detach().numpy(), return_tensors="pt") >>> doc_scores = torch.bmm( ... question_hidden_states.unsqueeze(1), docs_dict["retrieved_doc_embeds"].float().transpose(1, 2) - ... ).squeeze(1) + >>> ).squeeze(1) >>> # 3. Forward to generator >>> outputs = model( ... context_input_ids=docs_dict["context_input_ids"], diff --git a/src/transformers/models/rag/retrieval_rag.py b/src/transformers/models/rag/retrieval_rag.py index 7a3c5635f24f9..f39fc48d27c8b 100644 --- a/src/transformers/models/rag/retrieval_rag.py +++ b/src/transformers/models/rag/retrieval_rag.py @@ -354,7 +354,7 @@ class RagRetriever: >>> dataset = ( ... ... - ... ) # dataset must be a datasets.Datasets object with columns "title", "text" and "embeddings", and it must have a faiss index + >>> ) # dataset must be a datasets.Datasets object with columns "title", "text" and "embeddings", and it must have a faiss index >>> retriever = RagRetriever.from_pretrained("facebook/dpr-ctx_encoder-single-nq-base", indexed_dataset=dataset) >>> # To load your own indexed dataset built with the datasets library that was saved on disk. More info in examples/rag/use_own_knowledge_dataset.py diff --git a/src/transformers/models/realm/modeling_realm.py b/src/transformers/models/realm/modeling_realm.py index b2cf47e9faa37..0a27cd7399a37 100644 --- a/src/transformers/models/realm/modeling_realm.py +++ b/src/transformers/models/realm/modeling_realm.py @@ -1782,7 +1782,7 @@ def forward( ... add_special_tokens=False, ... return_token_type_ids=False, ... return_attention_mask=False, - ... ).input_ids + >>> ).input_ids >>> reader_output, predicted_answer_ids = model(**question_ids, answer_ids=answer_ids, return_dict=False) >>> predicted_answer = tokenizer.decode(predicted_answer_ids) diff --git a/src/transformers/models/speech_to_text/modeling_tf_speech_to_text.py b/src/transformers/models/speech_to_text/modeling_tf_speech_to_text.py index c78d19056bd30..8980636c3b32b 100755 --- a/src/transformers/models/speech_to_text/modeling_tf_speech_to_text.py +++ b/src/transformers/models/speech_to_text/modeling_tf_speech_to_text.py @@ -1387,7 +1387,7 @@ def call( >>> input_features = processor( ... ds["speech"][0], sampling_rate=16000, return_tensors="tf" - ... ).input_features # Batch size 1 + >>> ).input_features # Batch size 1 >>> generated_ids = model.generate(input_features) >>> transcription = processor.batch_decode(generated_ids) diff --git a/src/transformers/models/t5/modeling_flax_t5.py b/src/transformers/models/t5/modeling_flax_t5.py index 767caea3eb38f..263412578c23d 100644 --- a/src/transformers/models/t5/modeling_flax_t5.py +++ b/src/transformers/models/t5/modeling_flax_t5.py @@ -1344,7 +1344,7 @@ class FlaxT5Model(FlaxT5PreTrainedModel): >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="np" - ... ).input_ids + >>> ).input_ids >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="np").input_ids >>> # forward pass diff --git a/src/transformers/models/t5/modeling_t5.py b/src/transformers/models/t5/modeling_t5.py index 630e9dd17aa54..9b56e5ba519ac 100644 --- a/src/transformers/models/t5/modeling_t5.py +++ b/src/transformers/models/t5/modeling_t5.py @@ -1375,7 +1375,7 @@ def forward( >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="pt" - ... ).input_ids # Batch size 1 + >>> ).input_ids # Batch size 1 >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids # Batch size 1 >>> # forward pass @@ -1583,7 +1583,7 @@ def forward( >>> # inference >>> input_ids = tokenizer( ... "summarize: studies have shown that owning a dog is good for you", return_tensors="pt" - ... ).input_ids # Batch size 1 + >>> ).input_ids # Batch size 1 >>> outputs = model.generate(input_ids) >>> print(tokenizer.decode(outputs[0], skip_special_tokens=True)) >>> # studies have shown that owning a dog is good for you. @@ -1831,7 +1831,7 @@ def forward( >>> model = T5EncoderModel.from_pretrained("t5-small") >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="pt" - ... ).input_ids # Batch size 1 + >>> ).input_ids # Batch size 1 >>> outputs = model(input_ids=input_ids) >>> last_hidden_states = outputs.last_hidden_state ```""" diff --git a/src/transformers/models/t5/modeling_tf_t5.py b/src/transformers/models/t5/modeling_tf_t5.py index 3434a6ea4f377..2e48174a9048b 100644 --- a/src/transformers/models/t5/modeling_tf_t5.py +++ b/src/transformers/models/t5/modeling_tf_t5.py @@ -1165,7 +1165,7 @@ def call( >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="tf" - ... ).input_ids # Batch size 1 + >>> ).input_ids # Batch size 1 >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="tf").input_ids # Batch size 1 >>> # forward pass @@ -1353,7 +1353,7 @@ def call( >>> # inference >>> inputs = tokenizer( ... "summarize: studies have shown that owning a dog is good for you", return_tensors="tf" - ... ).input_ids # Batch size 1 + >>> ).input_ids # Batch size 1 >>> outputs = model.generate(inputs) >>> print(tokenizer.decode(outputs[0], skip_special_tokens=True)) >>> # studies have shown that owning a dog is good for you @@ -1642,7 +1642,7 @@ def call( >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="tf" - ... ).input_ids # Batch size 1 + >>> ).input_ids # Batch size 1 >>> outputs = model(input_ids) ```""" diff --git a/src/transformers/models/tapas/modeling_tapas.py b/src/transformers/models/tapas/modeling_tapas.py index b0c3786ca05a4..12703cc2dfaaa 100644 --- a/src/transformers/models/tapas/modeling_tapas.py +++ b/src/transformers/models/tapas/modeling_tapas.py @@ -1068,7 +1068,7 @@ def forward( ... ) >>> labels = tokenizer( ... table=table, queries="How many movies has George Clooney played in?", return_tensors="pt" - ... )["input_ids"] + >>> )["input_ids"] >>> outputs = model(**inputs, labels=labels) >>> logits = outputs.logits diff --git a/src/transformers/models/tapas/modeling_tf_tapas.py b/src/transformers/models/tapas/modeling_tf_tapas.py index d2da0644627ac..e91baaab8edb2 100644 --- a/src/transformers/models/tapas/modeling_tf_tapas.py +++ b/src/transformers/models/tapas/modeling_tf_tapas.py @@ -1095,7 +1095,7 @@ def call( ... ) >>> labels = tokenizer( ... table=table, queries="How many movies has George Clooney played in?", return_tensors="tf" - ... )["input_ids"] + >>> )["input_ids"] >>> outputs = model(**inputs, labels=labels) >>> logits = outputs.logits diff --git a/src/transformers/models/vision_encoder_decoder/modeling_tf_vision_encoder_decoder.py b/src/transformers/models/vision_encoder_decoder/modeling_tf_vision_encoder_decoder.py index 6bbf514091038..edc2973a07348 100644 --- a/src/transformers/models/vision_encoder_decoder/modeling_tf_vision_encoder_decoder.py +++ b/src/transformers/models/vision_encoder_decoder/modeling_tf_vision_encoder_decoder.py @@ -326,7 +326,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): >>> output_ids = model.generate( ... pixel_values, max_length=16, num_beams=4, return_dict_in_generate=True - ... ).sequences + >>> ).sequences >>> preds = decoder_tokenizer.batch_decode(output_ids, skip_special_tokens=True) >>> preds = [pred.strip() for pred in preds] diff --git a/src/transformers/models/wav2vec2/modeling_flax_wav2vec2.py b/src/transformers/models/wav2vec2/modeling_flax_wav2vec2.py index 7709e43ab9556..1386ca37b075d 100644 --- a/src/transformers/models/wav2vec2/modeling_flax_wav2vec2.py +++ b/src/transformers/models/wav2vec2/modeling_flax_wav2vec2.py @@ -1081,7 +1081,7 @@ class FlaxWav2Vec2Model(FlaxWav2Vec2PreTrainedModel): >>> input_values = processor( ... ds["speech"][0], sampling_rate=16_000, return_tensors="np" - ... ).input_values # Batch size 1 + >>> ).input_values # Batch size 1 >>> hidden_states = model(input_values).last_hidden_state ``` """ @@ -1200,7 +1200,7 @@ class FlaxWav2Vec2ForCTC(FlaxWav2Vec2PreTrainedModel): >>> input_values = processor( ... ds["speech"][0], sampling_rate=16_000, return_tensors="np" - ... ).input_values # Batch size 1 + >>> ).input_values # Batch size 1 >>> logits = model(input_values).logits >>> predicted_ids = jnp.argmax(logits, axis=-1) diff --git a/src/transformers/models/xlm/modeling_xlm.py b/src/transformers/models/xlm/modeling_xlm.py index ebb3c503475c0..ed0817afbb895 100755 --- a/src/transformers/models/xlm/modeling_xlm.py +++ b/src/transformers/models/xlm/modeling_xlm.py @@ -1039,7 +1039,7 @@ def forward( >>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze( ... 0 - ... ) # Batch size 1 + >>> ) # Batch size 1 >>> start_positions = torch.tensor([1]) >>> end_positions = torch.tensor([3]) diff --git a/src/transformers/models/xlm_prophetnet/modeling_xlm_prophetnet.py b/src/transformers/models/xlm_prophetnet/modeling_xlm_prophetnet.py index 8961fbbfc3740..dfb7b394915b5 100644 --- a/src/transformers/models/xlm_prophetnet/modeling_xlm_prophetnet.py +++ b/src/transformers/models/xlm_prophetnet/modeling_xlm_prophetnet.py @@ -98,7 +98,7 @@ class XLMProphetNetModel(ProphetNetModel): >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="pt" - ... ).input_ids # Batch size 1 + >>> ).input_ids # Batch size 1 >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids # Batch size 1 >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids) @@ -124,7 +124,7 @@ class XLMProphetNetForConditionalGeneration(ProphetNetForConditionalGeneration): >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="pt" - ... ).input_ids # Batch size 1 + >>> ).input_ids # Batch size 1 >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids # Batch size 1 >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids) diff --git a/src/transformers/models/xlnet/modeling_tf_xlnet.py b/src/transformers/models/xlnet/modeling_tf_xlnet.py index df4111d263172..f5a1cba3c837f 100644 --- a/src/transformers/models/xlnet/modeling_tf_xlnet.py +++ b/src/transformers/models/xlnet/modeling_tf_xlnet.py @@ -1281,17 +1281,17 @@ def call( >>> # We show how to setup inputs to predict a next token using a bi-directional context. >>> input_ids = tf.constant(tokenizer.encode("Hello, my dog is very ", add_special_tokens=True))[ ... None, : - ... ] # We will predict the masked token + >>> ] # We will predict the masked token >>> perm_mask = np.zeros((1, input_ids.shape[1], input_ids.shape[1])) >>> perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token >>> target_mapping = np.zeros( ... (1, 1, input_ids.shape[1]) - ... ) # Shape [1, 1, seq_length] => let's predict one token + >>> ) # Shape [1, 1, seq_length] => let's predict one token >>> target_mapping[ ... 0, 0, -1 - ... ] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) + >>> ] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) >>> outputs = model( ... input_ids, @@ -1301,7 +1301,7 @@ def call( >>> next_token_logits = outputs[ ... 0 - ... ] # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] + >>> ] # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] ```""" transformer_outputs = self.transformer( input_ids=input_ids, diff --git a/src/transformers/models/xlnet/modeling_xlnet.py b/src/transformers/models/xlnet/modeling_xlnet.py index dc7f78eeb8e2f..079c636628f2f 100755 --- a/src/transformers/models/xlnet/modeling_xlnet.py +++ b/src/transformers/models/xlnet/modeling_xlnet.py @@ -1400,47 +1400,47 @@ def forward( >>> # We show how to setup inputs to predict a next token using a bi-directional context. >>> input_ids = torch.tensor( ... tokenizer.encode("Hello, my dog is very ", add_special_tokens=False) - ... ).unsqueeze( + >>> ).unsqueeze( ... 0 - ... ) # We will predict the masked token + >>> ) # We will predict the masked token >>> perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float) >>> perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token >>> target_mapping = torch.zeros( ... (1, 1, input_ids.shape[1]), dtype=torch.float - ... ) # Shape [1, 1, seq_length] => let's predict one token + >>> ) # Shape [1, 1, seq_length] => let's predict one token >>> target_mapping[ ... 0, 0, -1 - ... ] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) + >>> ] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) >>> outputs = model(input_ids, perm_mask=perm_mask, target_mapping=target_mapping) >>> next_token_logits = outputs[ ... 0 - ... ] # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] + >>> ] # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] >>> # The same way can the XLNetLMHeadModel be used to be trained by standard auto-regressive language modeling. >>> input_ids = torch.tensor( ... tokenizer.encode("Hello, my dog is very ", add_special_tokens=False) - ... ).unsqueeze( + >>> ).unsqueeze( ... 0 - ... ) # We will predict the masked token + >>> ) # We will predict the masked token >>> labels = torch.tensor(tokenizer.encode("cute", add_special_tokens=False)).unsqueeze(0) >>> assert labels.shape[0] == 1, "only one word will be predicted" >>> perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float) >>> perm_mask[ ... :, :, -1 - ... ] = 1.0 # Previous tokens don't see last token as is done in standard auto-regressive lm training + >>> ] = 1.0 # Previous tokens don't see last token as is done in standard auto-regressive lm training >>> target_mapping = torch.zeros( ... (1, 1, input_ids.shape[1]), dtype=torch.float - ... ) # Shape [1, 1, seq_length] => let's predict one token + >>> ) # Shape [1, 1, seq_length] => let's predict one token >>> target_mapping[ ... 0, 0, -1 - ... ] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) + >>> ] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) >>> outputs = model(input_ids, perm_mask=perm_mask, target_mapping=target_mapping, labels=labels) >>> loss = outputs.loss >>> next_token_logits = ( ... outputs.logits - ... ) # Logits have shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] + >>> ) # Logits have shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] ```""" return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1980,7 +1980,7 @@ def forward( >>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze( ... 0 - ... ) # Batch size 1 + >>> ) # Batch size 1 >>> start_positions = torch.tensor([1]) >>> end_positions = torch.tensor([3]) >>> outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) From 096eb74a6ed28b5bc6ba5169134fefb4eb6c5aea Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Fri, 6 May 2022 12:24:45 +0200 Subject: [PATCH 024/145] make repo consistency --- docs/source/en/index.mdx | 2 +- src/transformers/utils/dummy_tokenizers_objects.py | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/docs/source/en/index.mdx b/docs/source/en/index.mdx index 6550f434da69c..16f1816b1dab6 100644 --- a/docs/source/en/index.mdx +++ b/docs/source/en/index.mdx @@ -229,7 +229,7 @@ Flax), PyTorch, and/or TensorFlow. | Nystromformer | ❌ | ❌ | ✅ | ❌ | ❌ | | OpenAI GPT | ✅ | ✅ | ✅ | ✅ | ❌ | | OpenAI GPT-2 | ✅ | ✅ | ✅ | ✅ | ✅ | -| OPT | ✅ | ✅ | ✅ | ✅ | ✅ | +| OPT | ❌ | ❌ | ✅ | ✅ | ✅ | | Pegasus | ✅ | ✅ | ✅ | ✅ | ✅ | | Perceiver | ✅ | ❌ | ✅ | ❌ | ❌ | | PLBart | ✅ | ❌ | ✅ | ❌ | ❌ | diff --git a/src/transformers/utils/dummy_tokenizers_objects.py b/src/transformers/utils/dummy_tokenizers_objects.py index 9971577e8089f..12cec6a4a2605 100644 --- a/src/transformers/utils/dummy_tokenizers_objects.py +++ b/src/transformers/utils/dummy_tokenizers_objects.py @@ -241,13 +241,6 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["tokenizers"]) -class OPTTokenizerFast(metaclass=DummyObject): - _backends = ["tokenizers"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["tokenizers"]) - - class PegasusTokenizerFast(metaclass=DummyObject): _backends = ["tokenizers"] From 020843a6666e1df4de1c77c3a3dcc13a974e6e08 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Fri, 6 May 2022 12:30:09 +0200 Subject: [PATCH 025/145] Removed PretrainedOPTModel --- docs/source/en/model_doc/opt.mdx | 5 ----- src/transformers/__init__.py | 2 -- src/transformers/models/opt/__init__.py | 2 -- src/transformers/utils/dummy_pt_objects.py | 7 ------- 4 files changed, 16 deletions(-) diff --git a/docs/source/en/model_doc/opt.mdx b/docs/source/en/model_doc/opt.mdx index 63e478cedbaae..f82900d2b3d04 100644 --- a/docs/source/en/model_doc/opt.mdx +++ b/docs/source/en/model_doc/opt.mdx @@ -45,11 +45,6 @@ The original code can be found [here](). [[autodoc]] OPTPretrainedModel - all -## PretrainedOPTModel - -[[autodoc]] PretrainedOPTModel - - all - ## OPTForConditionalGeneration [[autodoc]] OPTForConditionalGeneration diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 89d91fd65d8ee..48526495bae0f 100755 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -1276,7 +1276,6 @@ "OPTForCausalLM", "OPTModel", "OPTPretrainedModel", - "PretrainedOPTModel", ] ) _import_structure["models.pegasus"].extend( @@ -3543,7 +3542,6 @@ OPTForCausalLM, OPTModel, OPTPretrainedModel, - PretrainedOPTModel, ) from .models.pegasus import ( PegasusForCausalLM, diff --git a/src/transformers/models/opt/__init__.py b/src/transformers/models/opt/__init__.py index eef337d3bbd93..f36c6bf9faca2 100644 --- a/src/transformers/models/opt/__init__.py +++ b/src/transformers/models/opt/__init__.py @@ -31,7 +31,6 @@ "OPTForCausalLM", "OPTModel", "OPTPretrainedModel", - "PretrainedOPTModel", ] if is_tf_available(): @@ -57,7 +56,6 @@ OPTForCausalLM, OPTModel, OPTPretrainedModel, - PretrainedOPTModel, ) if is_tf_available(): diff --git a/src/transformers/utils/dummy_pt_objects.py b/src/transformers/utils/dummy_pt_objects.py index cbe4364466d85..b0f19b0070436 100644 --- a/src/transformers/utils/dummy_pt_objects.py +++ b/src/transformers/utils/dummy_pt_objects.py @@ -2962,13 +2962,6 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["torch"]) -class PretrainedOPTModel(metaclass=DummyObject): - _backends = ["torch"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["torch"]) - - class PegasusForCausalLM(metaclass=DummyObject): _backends = ["torch"] From c63d9f8757efb80c3e2f3068ad8d44fec7997bd9 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Fri, 6 May 2022 13:24:41 +0200 Subject: [PATCH 026/145] fix opt.mdx removed other heads --- docs/source/en/model_doc/opt.mdx | 38 -------------------------------- 1 file changed, 38 deletions(-) diff --git a/docs/source/en/model_doc/opt.mdx b/docs/source/en/model_doc/opt.mdx index f82900d2b3d04..5c117193e987f 100644 --- a/docs/source/en/model_doc/opt.mdx +++ b/docs/source/en/model_doc/opt.mdx @@ -45,20 +45,6 @@ The original code can be found [here](). [[autodoc]] OPTPretrainedModel - all -## OPTForConditionalGeneration - -[[autodoc]] OPTForConditionalGeneration - - forward - -## OPTForSequenceClassification - -[[autodoc]] OPTForSequenceClassification - - forward - -## OPTForQuestionAnswering - -[[autodoc]] OPTForQuestionAnswering - - forward ## OPTForCausalLM @@ -75,10 +61,6 @@ The original code can be found [here](). [[autodoc]] TFOPTPretrainedModel - call -## TFOPTForConditionalGeneration - -[[autodoc]] TFOPTForConditionalGeneration - - call ## FlaxOPTModel @@ -87,26 +69,6 @@ The original code can be found [here](). - encode - decode -## FlaxOPTForConditionalGeneration - -[[autodoc]] FlaxOPTForConditionalGeneration - - __call__ - - encode - - decode - -## FlaxOPTForSequenceClassification - -[[autodoc]] FlaxOPTForSequenceClassification - - __call__ - - encode - - decode - -## FlaxOPTForQuestionAnswering - -[[autodoc]] FlaxOPTForQuestionAnswering - - __call__ - - encode - - decode ## FlaxOPTForCausalLM From 8b6e4968fe87a51f46a3c534f6a0f14886ab09f3 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Fri, 6 May 2022 13:26:12 +0200 Subject: [PATCH 027/145] fix init, removed 3 heads --- src/transformers/__init__.py | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 48526495bae0f..a7684a1315e82 100755 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -2082,7 +2082,7 @@ "TFOpenAIGPTPreTrainedModel", ] ) - _import_structure["models.opt"].extend(["TFOPTForConditionalGeneration", "TFOPTModel", "TFOPTPretrainedModel"]) + _import_structure["models.opt"].extend(["TFOPTModel", "TFOPTPretrainedModel"]) _import_structure["models.pegasus"].extend( ["TFPegasusForConditionalGeneration", "TFPegasusModel", "TFPegasusPreTrainedModel"] ) @@ -2420,9 +2420,6 @@ [ "FlaxOPTDecoderPreTrainedModel", "FlaxOPTForCausalLM", - "FlaxOPTForConditionalGeneration", - "FlaxOPTForQuestionAnswering", - "FlaxOPTForSequenceClassification", "FlaxOPTModel", "FlaxOPTPreTrainedModel", ] @@ -3537,12 +3534,7 @@ OpenAIGPTPreTrainedModel, load_tf_weights_in_openai_gpt, ) - from .models.opt import ( - OPT_PRETRAINED_MODEL_ARCHIVE_LIST, - OPTForCausalLM, - OPTModel, - OPTPretrainedModel, - ) + from .models.opt import OPT_PRETRAINED_MODEL_ARCHIVE_LIST, OPTForCausalLM, OPTModel, OPTPretrainedModel from .models.pegasus import ( PegasusForCausalLM, PegasusForConditionalGeneration, @@ -4209,7 +4201,7 @@ TFOpenAIGPTModel, TFOpenAIGPTPreTrainedModel, ) - from .models.opt import TFOPTForConditionalGeneration, TFOPTModel, TFOPTPretrainedModel + from .models.opt import TFOPTModel, TFOPTPretrainedModel from .models.pegasus import TFPegasusForConditionalGeneration, TFPegasusModel, TFPegasusPreTrainedModel from .models.rag import TFRagModel, TFRagPreTrainedModel, TFRagSequenceForGeneration, TFRagTokenForGeneration from .models.rembert import ( @@ -4469,15 +4461,7 @@ FlaxMBartPreTrainedModel, ) from .models.mt5 import FlaxMT5ForConditionalGeneration, FlaxMT5Model - from .models.opt import ( - FlaxOPTDecoderPreTrainedModel, - FlaxOPTForCausalLM, - FlaxOPTForConditionalGeneration, - FlaxOPTForQuestionAnswering, - FlaxOPTForSequenceClassification, - FlaxOPTModel, - FlaxOPTPreTrainedModel, - ) + from .models.opt import FlaxOPTDecoderPreTrainedModel, FlaxOPTForCausalLM, FlaxOPTModel, FlaxOPTPreTrainedModel from .models.pegasus import FlaxPegasusForConditionalGeneration, FlaxPegasusModel, FlaxPegasusPreTrainedModel from .models.roberta import ( FlaxRobertaForCausalLM, From 0303f2bcd396a626cb8f354929939d81520570b5 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Fri, 6 May 2022 13:26:56 +0200 Subject: [PATCH 028/145] removed heads --- src/transformers/models/auto/modeling_flax_auto.py | 5 ----- src/transformers/models/auto/modeling_tf_auto.py | 2 -- 2 files changed, 7 deletions(-) diff --git a/src/transformers/models/auto/modeling_flax_auto.py b/src/transformers/models/auto/modeling_flax_auto.py index 1c6972eea5b1a..219d3b861d661 100644 --- a/src/transformers/models/auto/modeling_flax_auto.py +++ b/src/transformers/models/auto/modeling_flax_auto.py @@ -66,7 +66,6 @@ ("bert", "FlaxBertForPreTraining"), ("big_bird", "FlaxBigBirdForPreTraining"), ("bart", "FlaxBartForConditionalGeneration"), - ("opt", "FlaxOPTForConditionalGeneration"), ("electra", "FlaxElectraForPreTraining"), ("mbart", "FlaxMBartForConditionalGeneration"), ("t5", "FlaxT5ForConditionalGeneration"), @@ -86,7 +85,6 @@ ("bert", "FlaxBertForMaskedLM"), ("big_bird", "FlaxBigBirdForMaskedLM"), ("bart", "FlaxBartForConditionalGeneration"), - ("opt", "FlaxOPTForConditionalGeneration"), ("electra", "FlaxElectraForMaskedLM"), ("mbart", "FlaxMBartForConditionalGeneration"), ("roformer", "FlaxRoFormerForMaskedLM"), @@ -99,7 +97,6 @@ ("blenderbot-small", "FlaxBlenderbotSmallForConditionalGeneration"), ("pegasus", "FlaxPegasusForConditionalGeneration"), ("bart", "FlaxBartForConditionalGeneration"), - ("opt", "FlaxOPTForConditionalGeneration"), ("mbart", "FlaxMBartForConditionalGeneration"), ("t5", "FlaxT5ForConditionalGeneration"), ("mt5", "FlaxMT5ForConditionalGeneration"), @@ -149,7 +146,6 @@ ("bert", "FlaxBertForSequenceClassification"), ("big_bird", "FlaxBigBirdForSequenceClassification"), ("bart", "FlaxBartForSequenceClassification"), - ("opt", "FlaxOPTForSequenceClassification"), ("electra", "FlaxElectraForSequenceClassification"), ("mbart", "FlaxMBartForSequenceClassification"), ("roformer", "FlaxRoFormerForSequenceClassification"), @@ -166,7 +162,6 @@ ("bert", "FlaxBertForQuestionAnswering"), ("big_bird", "FlaxBigBirdForQuestionAnswering"), ("bart", "FlaxBartForQuestionAnswering"), - ("opt", "FlaxOPTForQuestionAnswering"), ("electra", "FlaxElectraForQuestionAnswering"), ("mbart", "FlaxMBartForQuestionAnswering"), ("roformer", "FlaxRoFormerForQuestionAnswering"), diff --git a/src/transformers/models/auto/modeling_tf_auto.py b/src/transformers/models/auto/modeling_tf_auto.py index 334900a0d2ee6..815e47e1ff7df 100644 --- a/src/transformers/models/auto/modeling_tf_auto.py +++ b/src/transformers/models/auto/modeling_tf_auto.py @@ -85,7 +85,6 @@ ("distilbert", "TFDistilBertForMaskedLM"), ("albert", "TFAlbertForPreTraining"), ("bart", "TFBartForConditionalGeneration"), - ("opt", "TFOPTForConditionalGeneration"), ("camembert", "TFCamembertForMaskedLM"), ("xlm-roberta", "TFXLMRobertaForMaskedLM"), ("roberta", "TFRobertaForMaskedLM"), @@ -212,7 +211,6 @@ ("blenderbot", "TFBlenderbotForConditionalGeneration"), ("blenderbot-small", "TFBlenderbotSmallForConditionalGeneration"), ("bart", "TFBartForConditionalGeneration"), - ("opt", "TFOPTForConditionalGeneration"), ("encoder-decoder", "TFEncoderDecoderModel"), ] ) From 2c0327d84aa1db561d259e18d195c4dd16cee0f8 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Fri, 6 May 2022 13:27:59 +0200 Subject: [PATCH 029/145] finished cleaning head --- src/transformers/models/opt/__init__.py | 17 +++-------------- ...pt_original_pytorch_checkpoint_to_pytorch.py | 3 +-- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/src/transformers/models/opt/__init__.py b/src/transformers/models/opt/__init__.py index f36c6bf9faca2..7192f1255d079 100644 --- a/src/transformers/models/opt/__init__.py +++ b/src/transformers/models/opt/__init__.py @@ -34,15 +34,12 @@ ] if is_tf_available(): - _import_structure["modeling_tf_opt"] = ["TFOPTForConditionalGeneration", "TFOPTModel", "TFOPTPretrainedModel"] + _import_structure["modeling_tf_opt"] = ["TFOPTModel", "TFOPTPretrainedModel"] if is_flax_available(): _import_structure["modeling_flax_opt"] = [ "FlaxOPTDecoderPreTrainedModel", "FlaxOPTForCausalLM", - "FlaxOPTForConditionalGeneration", - "FlaxOPTForQuestionAnswering", - "FlaxOPTForSequenceClassification", "FlaxOPTModel", "FlaxOPTPreTrainedModel", ] @@ -51,23 +48,15 @@ from .configuration_opt import OPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OPTConfig, OPTOnnxConfig if is_torch_available(): - from .modeling_opt import ( - OPT_PRETRAINED_MODEL_ARCHIVE_LIST, - OPTForCausalLM, - OPTModel, - OPTPretrainedModel, - ) + from .modeling_opt import OPT_PRETRAINED_MODEL_ARCHIVE_LIST, OPTForCausalLM, OPTModel, OPTPretrainedModel if is_tf_available(): - from .modeling_tf_opt import TFOPTForConditionalGeneration, TFOPTModel, TFOPTPretrainedModel + from .modeling_tf_opt import TFOPTModel, TFOPTPretrainedModel if is_flax_available(): from .modeling_flax_opt import ( FlaxOPTDecoderPreTrainedModel, FlaxOPTForCausalLM, - FlaxOPTForConditionalGeneration, - FlaxOPTForQuestionAnswering, - FlaxOPTForSequenceClassification, FlaxOPTModel, FlaxOPTPreTrainedModel, ) diff --git a/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py index 1ce87d3b72962..ac29b24a5b7cc 100644 --- a/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py @@ -22,12 +22,11 @@ import torch from torch import nn -from transformers import OPTConfig, OPTForSequenceClassification, OPTModel +from transformers import OPTConfig, OPTModel from transformers.utils import logging FAIRSEQ_MODELS = ["opt.large", "opt.large.mnli", "opt.large.cnn", "opt_xsum/model.pt"] -extra_arch = {"opt.large": OPTModel, "opt.large.mnli": OPTForSequenceClassification} # if version.parse(fairseq.__version__) < version.parse("0.9.0"): # raise Exception("requires fairseq >= 0.9.0") From 4aa6ab2b8bf5684cb2f90f7b1316f06f4b74a54a Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Fri, 6 May 2022 13:33:05 +0200 Subject: [PATCH 030/145] removed seauence classif and question answering --- .../models/opt/modeling_flax_opt.py | 509 +----------------- 1 file changed, 2 insertions(+), 507 deletions(-) diff --git a/src/transformers/models/opt/modeling_flax_opt.py b/src/transformers/models/opt/modeling_flax_opt.py index f58775a5f42a6..cd75183f071a2 100644 --- a/src/transformers/models/opt/modeling_flax_opt.py +++ b/src/transformers/models/opt/modeling_flax_opt.py @@ -17,7 +17,7 @@ import math import random from functools import partial -from typing import Callable, Optional, Tuple +from typing import Optional, Tuple import numpy as np @@ -35,18 +35,9 @@ FlaxBaseModelOutput, FlaxBaseModelOutputWithPastAndCrossAttentions, FlaxCausalLMOutputWithCrossAttentions, - FlaxSeq2SeqLMOutput, FlaxSeq2SeqModelOutput, - FlaxSeq2SeqQuestionAnsweringModelOutput, - FlaxSeq2SeqSequenceClassifierOutput, -) -from ...modeling_flax_utils import ( - ACT2FN, - FlaxPreTrainedModel, - append_call_sample_docstring, - append_replace_return_docstrings, - overwrite_call_docstring, ) +from ...modeling_flax_utils import ACT2FN, FlaxPreTrainedModel, append_call_sample_docstring from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings from .configuration_opt import OPTConfig @@ -1259,502 +1250,6 @@ class FlaxOPTModel(FlaxOPTPreTrainedModel): FlaxOPTModel, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSeq2SeqModelOutput, _CONFIG_FOR_DOC ) - -# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartForConditionalGenerationModule with Bart->OPT -class FlaxOPTForConditionalGenerationModule(nn.Module): - config: OPTConfig - dtype: jnp.dtype = jnp.float32 - bias_init: Callable[..., jnp.ndarray] = jax.nn.initializers.zeros - - def setup(self): - self.model = FlaxOPTModule(config=self.config, dtype=self.dtype) - self.lm_head = nn.Dense( - self.model.shared.num_embeddings, - use_bias=False, - dtype=self.dtype, - kernel_init=jax.nn.initializers.normal(self.config.init_std), - ) - self.final_logits_bias = self.param("final_logits_bias", self.bias_init, (1, self.model.shared.num_embeddings)) - - def _get_encoder_module(self): - return self.model.encoder - - def _get_decoder_module(self): - return self.model.decoder - - def __call__( - self, - input_ids, - attention_mask, - decoder_input_ids, - decoder_attention_mask, - position_ids, - decoder_position_ids, - output_attentions: bool = False, - output_hidden_states: bool = False, - return_dict: bool = True, - deterministic: bool = True, - ): - outputs = self.model( - input_ids=input_ids, - attention_mask=attention_mask, - decoder_input_ids=decoder_input_ids, - decoder_attention_mask=decoder_attention_mask, - position_ids=position_ids, - decoder_position_ids=decoder_position_ids, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - deterministic=deterministic, - ) - - hidden_states = outputs[0] - - if self.config.tie_word_embeddings: - shared_embedding = self.model.variables["params"]["shared"]["embedding"] - lm_logits = self.lm_head.apply({"params": {"kernel": shared_embedding.T}}, hidden_states) - else: - lm_logits = self.lm_head(hidden_states) - - lm_logits += jax.lax.stop_gradient(self.final_logits_bias.astype(self.dtype)) - - if not return_dict: - output = (lm_logits,) + outputs[1:] - return output - - return FlaxSeq2SeqLMOutput( - logits=lm_logits, - decoder_hidden_states=outputs.decoder_hidden_states, - decoder_attentions=outputs.decoder_attentions, - cross_attentions=outputs.cross_attentions, - encoder_last_hidden_state=outputs.encoder_last_hidden_state, - encoder_hidden_states=outputs.encoder_hidden_states, - encoder_attentions=outputs.encoder_attentions, - ) - - -@add_start_docstrings( - "The OPT Model with a language modeling head. Can be used for summarization.", OPT_START_DOCSTRING -) -# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartForConditionalGeneration with BART->OPT,Bart->OPT,bart->opt -class FlaxOPTForConditionalGeneration(FlaxOPTPreTrainedModel): - module_class = FlaxOPTForConditionalGenerationModule - dtype: jnp.dtype = jnp.float32 - - @add_start_docstrings(OPT_DECODE_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=FlaxCausalLMOutputWithCrossAttentions, config_class=OPTConfig) - def decode( - self, - decoder_input_ids, - encoder_outputs, - encoder_attention_mask: Optional[jnp.ndarray] = None, - decoder_attention_mask: Optional[jnp.ndarray] = None, - decoder_position_ids: Optional[jnp.ndarray] = None, - past_key_values: dict = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - train: bool = False, - params: dict = None, - dropout_rng: PRNGKey = None, - ): - r""" - Returns: - - Example: - - ```python - >>> import jax.numpy as jnp - >>> from transformers import OPTTokenizer, FlaxOPTForConditionalGeneration - - >>> model = FlaxOPTForConditionalGeneration.from_pretrained("facebook/opt-large-cnn") - >>> tokenizer = OPTTokenizer.from_pretrained("facebook/opt-large-cnn") - - >>> text = "My friends are cool but they eat too many carbs." - >>> inputs = tokenizer(text, max_length=1024, return_tensors="jax") - >>> encoder_outputs = model.encode(**inputs) - - >>> decoder_start_token_id = model.config.decoder_start_token_id - >>> decoder_input_ids = jnp.ones((inputs.input_ids.shape[0], 1), dtype="i4") * decoder_start_token_id - - >>> outputs = model.decode(decoder_input_ids, encoder_outputs) - >>> logits = outputs.logits - ```""" - output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions - output_hidden_states = ( - output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states - ) - return_dict = return_dict if return_dict is not None else self.config.return_dict - - encoder_hidden_states = encoder_outputs[0] - if encoder_attention_mask is None: - batch_size, sequence_length = encoder_hidden_states.shape[:2] - encoder_attention_mask = jnp.ones((batch_size, sequence_length)) - - batch_size, sequence_length = decoder_input_ids.shape - if decoder_attention_mask is None: - decoder_attention_mask = jnp.ones((batch_size, sequence_length)) - - if decoder_position_ids is None: - if past_key_values is not None: - raise ValueError("Make sure to provide `decoder_position_ids` when passing `past_key_values`.") - - decoder_position_ids = jnp.broadcast_to( - jnp.arange(sequence_length)[None, :], (batch_size, sequence_length) - ) - - # Handle any PRNG if needed - rngs = {} - if dropout_rng is not None: - rngs["dropout"] = dropout_rng - - inputs = {"params": params or self.params} - - # if past_key_values are passed then cache is already initialized a private flag init_cache has to be - # passed down to ensure cache is used. It has to be made sure that cache is marked as mutable so that - # it can be changed by FlaxOPTAttention module - if past_key_values: - inputs["cache"] = past_key_values - mutable = ["cache"] - else: - mutable = False - - def _decoder_forward(module, decoder_input_ids, decoder_attention_mask, decoder_position_ids, **kwargs): - decoder_module = module._get_decoder_module() - outputs = decoder_module( - decoder_input_ids, - decoder_attention_mask, - decoder_position_ids, - **kwargs, - ) - hidden_states = outputs[0] - - if self.config.tie_word_embeddings: - shared_embedding = module.model.variables["params"]["shared"]["embedding"] - lm_logits = module.lm_head.apply({"params": {"kernel": shared_embedding.T}}, hidden_states) - else: - lm_logits = module.lm_head(hidden_states) - - lm_logits += module.final_logits_bias.astype(self.dtype) - return lm_logits, outputs - - outputs = self.module.apply( - inputs, - decoder_input_ids=jnp.array(decoder_input_ids, dtype="i4"), - decoder_attention_mask=jnp.array(decoder_attention_mask, dtype="i4"), - decoder_position_ids=jnp.array(decoder_position_ids, dtype="i4"), - encoder_hidden_states=encoder_hidden_states, - encoder_attention_mask=jnp.array(encoder_attention_mask, dtype="i4"), - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - deterministic=not train, - rngs=rngs, - mutable=mutable, - method=_decoder_forward, - ) - - if past_key_values is None: - lm_logits, decoder_outputs = outputs - else: - (lm_logits, decoder_outputs), past = outputs - - if return_dict: - outputs = FlaxCausalLMOutputWithCrossAttentions( - logits=lm_logits, - hidden_states=decoder_outputs.hidden_states, - attentions=decoder_outputs.attentions, - cross_attentions=decoder_outputs.cross_attentions, - ) - else: - outputs = (lm_logits,) + decoder_outputs[1:] - - # add updated cache to model output - if past_key_values is not None and return_dict: - outputs["past_key_values"] = unfreeze(past["cache"]) - return outputs - elif past_key_values is not None and not return_dict: - outputs = outputs[:1] + (unfreeze(past["cache"]),) + outputs[1:] - - return outputs - - def prepare_inputs_for_generation( - self, - decoder_input_ids, - max_length, - attention_mask: Optional[jnp.DeviceArray] = None, - decoder_attention_mask: Optional[jnp.DeviceArray] = None, - encoder_outputs=None, - **kwargs - ): - # initializing the cache - batch_size, seq_length = decoder_input_ids.shape - - past_key_values = self.init_cache(batch_size, max_length, encoder_outputs) - # Note that usually one would have to put 0's in the attention_mask for x > input_ids.shape[-1] and x < cache_length. - # But since the decoder uses a causal mask, those positions are masked anyways. - # Thus we can create a single static attention_mask here, which is more efficient for compilation - extended_attention_mask = jnp.ones((batch_size, max_length), dtype="i4") - if decoder_attention_mask is not None: - position_ids = decoder_attention_mask.cumsum(axis=-1) - 1 - extended_attention_mask = lax.dynamic_update_slice(extended_attention_mask, decoder_attention_mask, (0, 0)) - else: - position_ids = jnp.broadcast_to(jnp.arange(seq_length, dtype="i4")[None, :], (batch_size, seq_length)) - - return { - "past_key_values": past_key_values, - "encoder_outputs": encoder_outputs, - "encoder_attention_mask": attention_mask, - "decoder_attention_mask": extended_attention_mask, - "decoder_position_ids": position_ids, - } - - def update_inputs_for_generation(self, model_outputs, model_kwargs): - model_kwargs["past_key_values"] = model_outputs.past_key_values - model_kwargs["decoder_position_ids"] = model_kwargs["decoder_position_ids"][:, -1:] + 1 - return model_kwargs - - -FLAX_OPT_CONDITIONAL_GENERATION_DOCSTRING = """ - Returns: - - Summarization example: - - ```python - >>> from transformers import OPTTokenizer, FlaxOPTForConditionalGeneration - - >>> model = FlaxOPTForConditionalGeneration.from_pretrained("facebook/opt-large-cnn") - >>> tokenizer = OPTTokenizer.from_pretrained("facebook/opt-large-cnn") - - >>> ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs." - >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors="np") - - >>> # Generate Summary - >>> summary_ids = model.generate(inputs["input_ids"]).sequences - >>> print(tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)) - ``` - - Mask filling example: - - ```python - >>> import jax - >>> from transformers import OPTTokenizer, FlaxOPTForConditionalGeneration - - >>> model = FlaxOPTForConditionalGeneration.from_pretrained("facebook/opt-large") - >>> tokenizer = OPTTokenizer.from_pretrained("facebook/opt-large") - - >>> TXT = "My friends are but they eat too many carbs." - >>> input_ids = tokenizer([TXT], return_tensors="jax")["input_ids"] - - >>> logits = model(input_ids).logits - >>> masked_index = (input_ids[0] == tokenizer.mask_token_id).nonzero()[0].item() - >>> probs = jax.nn.softmax(logits[0, masked_index], axis=0) - >>> values, predictions = jax.lax.top_k(probs, k=1) - - >>> tokenizer.decode(predictions).split() - ``` -""" - -overwrite_call_docstring( - FlaxOPTForConditionalGeneration, OPT_INPUTS_DOCSTRING + FLAX_OPT_CONDITIONAL_GENERATION_DOCSTRING -) -append_replace_return_docstrings( - FlaxOPTForConditionalGeneration, output_type=FlaxSeq2SeqLMOutput, config_class=_CONFIG_FOR_DOC -) - - -# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartForSequenceClassificationModule with Bart->OPT -class FlaxOPTForSequenceClassificationModule(nn.Module): - config: OPTConfig - dtype: jnp.dtype = jnp.float32 - num_labels: Optional[int] = None - - def setup(self): - self.model = FlaxOPTModule(config=self.config, dtype=self.dtype) - self.classification_head = FlaxOPTClassificationHead( - config=self.config, - inner_dim=self.config.d_model, - num_classes=self.num_labels if self.num_labels is not None else self.config.num_labels, - pooler_dropout=self.config.classifier_dropout, - ) - - def _get_encoder_module(self): - return self.model.encoder - - def _get_decoder_module(self): - return self.model.decoder - - def __call__( - self, - input_ids, - attention_mask, - decoder_input_ids, - decoder_attention_mask, - position_ids, - decoder_position_ids, - output_attentions: bool = False, - output_hidden_states: bool = False, - return_dict: bool = True, - deterministic: bool = True, - ): - outputs = self.model( - input_ids=input_ids, - attention_mask=attention_mask, - decoder_input_ids=decoder_input_ids, - decoder_attention_mask=decoder_attention_mask, - position_ids=position_ids, - decoder_position_ids=decoder_position_ids, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - deterministic=deterministic, - ) - - hidden_states = outputs[0] # last hidden state - - eos_mask = jnp.where(input_ids == self.config.eos_token_id, 1, 0) - - # The first condition is necessary to overcome jax._src.errors.ConcretizationTypeError during JIT compilation - if type(eos_mask) != jax.interpreters.partial_eval.DynamicJaxprTracer: - if len(jnp.unique(eos_mask.sum(1))) > 1: - raise ValueError("All examples must have the same number of tokens.") - - if any(eos_mask.sum(1) == 0): - raise ValueError("There are missing tokens in input_ids") - - # Ensure to keep 1 only for the last token for each example - eos_mask_noised = eos_mask + jnp.arange(eos_mask.shape[1]) * 1e-6 - eos_mask = jnp.where(eos_mask_noised == eos_mask_noised.max(1).reshape(-1, 1), 1, 0) - - sentence_representation = jnp.einsum("ijk, ij -> ijk", hidden_states, eos_mask).sum(1) - logits = self.classification_head(sentence_representation, deterministic=deterministic) - - if not return_dict: - output = (logits,) + outputs[1:] - return output - - return FlaxSeq2SeqSequenceClassifierOutput( - logits=logits, - decoder_hidden_states=outputs.decoder_hidden_states, - decoder_attentions=outputs.decoder_attentions, - cross_attentions=outputs.cross_attentions, - encoder_last_hidden_state=outputs.encoder_last_hidden_state, - encoder_hidden_states=outputs.encoder_hidden_states, - encoder_attentions=outputs.encoder_attentions, - ) - - -@add_start_docstrings( - """ - OPT model with a sequence classification/head on top (a linear layer on top of the pooled output) e.g. for GLUE - tasks. - """, - OPT_START_DOCSTRING, -) -# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartForSequenceClassification with Bart->OPT -class FlaxOPTForSequenceClassification(FlaxOPTPreTrainedModel): - module_class = FlaxOPTForSequenceClassificationModule - dtype = jnp.float32 - - -append_call_sample_docstring( - FlaxOPTForSequenceClassification, - _TOKENIZER_FOR_DOC, - _CHECKPOINT_FOR_DOC, - FlaxSeq2SeqSequenceClassifierOutput, - _CONFIG_FOR_DOC, -) - - -# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartForQuestionAnsweringModule with Bart->OPT -class FlaxOPTForQuestionAnsweringModule(nn.Module): - config: OPTConfig - dtype: jnp.dtype = jnp.float32 - num_labels = 2 - - def setup(self): - self.model = FlaxOPTModule(config=self.config, dtype=self.dtype) - self.qa_outputs = nn.Dense( - self.num_labels, dtype=self.dtype, kernel_init=jax.nn.initializers.normal(self.config.init_std) - ) - - def _get_encoder_module(self): - return self.model.encoder - - def _get_decoder_module(self): - return self.model.decoder - - def __call__( - self, - input_ids, - attention_mask, - decoder_input_ids, - decoder_attention_mask, - position_ids, - decoder_position_ids, - output_attentions: bool = False, - output_hidden_states: bool = False, - return_dict: bool = True, - deterministic: bool = True, - ): - outputs = self.model( - input_ids=input_ids, - attention_mask=attention_mask, - decoder_input_ids=decoder_input_ids, - decoder_attention_mask=decoder_attention_mask, - position_ids=position_ids, - decoder_position_ids=decoder_position_ids, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - deterministic=deterministic, - ) - - sequence_output = outputs[0] - - logits = self.qa_outputs(sequence_output) - start_logits, end_logits = jnp.split(logits, logits.shape[-1], axis=-1) - start_logits = start_logits.squeeze(-1) - end_logits = end_logits.squeeze(-1) - - if not return_dict: - output = (start_logits, end_logits) + outputs[1:] - return output - - return FlaxSeq2SeqQuestionAnsweringModelOutput( - start_logits=start_logits, - end_logits=end_logits, - decoder_hidden_states=outputs.decoder_hidden_states, - decoder_attentions=outputs.decoder_attentions, - cross_attentions=outputs.cross_attentions, - encoder_last_hidden_state=outputs.encoder_last_hidden_state, - encoder_hidden_states=outputs.encoder_hidden_states, - encoder_attentions=outputs.encoder_attentions, - ) - - -@add_start_docstrings( - """ - OPT Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layer - on top of the hidden-states output to compute `span start logits` and `span end logits`). - """, - OPT_START_DOCSTRING, -) -# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartForQuestionAnswering with Bart->OPT -class FlaxOPTForQuestionAnswering(FlaxOPTPreTrainedModel): - module_class = FlaxOPTForQuestionAnsweringModule - dtype = jnp.float32 - - -append_call_sample_docstring( - FlaxOPTForQuestionAnswering, - _TOKENIZER_FOR_DOC, - _CHECKPOINT_FOR_DOC, - FlaxSeq2SeqQuestionAnsweringModelOutput, - _CONFIG_FOR_DOC, -) - - # Copied from transformers.models.bart.modeling_flax_bart.FlaxBartDecoderPreTrainedModel with BART->OPT,Bart->OPT class FlaxOPTDecoderPreTrainedModel(FlaxPreTrainedModel): config_class = OPTConfig From 752f5123ef2f16650b696e565396f5f82e616ae0 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Fri, 6 May 2022 13:33:24 +0200 Subject: [PATCH 031/145] removed unused imports --- .../models/opt/modeling_tf_opt.py | 201 +----------------- 1 file changed, 2 insertions(+), 199 deletions(-) diff --git a/src/transformers/models/opt/modeling_tf_opt.py b/src/transformers/models/opt/modeling_tf_opt.py index d668f5a96abc2..e2685799ddad0 100644 --- a/src/transformers/models/opt/modeling_tf_opt.py +++ b/src/transformers/models/opt/modeling_tf_opt.py @@ -22,17 +22,11 @@ import tensorflow as tf from ...activations_tf import get_tf_activation -from ...modeling_tf_outputs import ( - TFBaseModelOutput, - TFBaseModelOutputWithPastAndCrossAttentions, - TFSeq2SeqLMOutput, - TFSeq2SeqModelOutput, -) +from ...modeling_tf_outputs import TFBaseModelOutput, TFBaseModelOutputWithPastAndCrossAttentions, TFSeq2SeqModelOutput # Public API from ...modeling_tf_utils import ( DUMMY_INPUTS, - TFCausalLanguageModelingLoss, TFModelInputType, TFPreTrainedModel, TFSharedEmbeddings, @@ -41,14 +35,7 @@ unpack_inputs, ) from ...tf_utils import shape_list, stable_softmax -from ...utils import ( - add_code_sample_docstrings, - add_end_docstrings, - add_start_docstrings, - add_start_docstrings_to_model_forward, - logging, - replace_return_docstrings, -) +from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging from .configuration_opt import OPTConfig @@ -1226,187 +1213,3 @@ def serving_output(self, output): encoder_hidden_states=enc_hs, encoder_attentions=enc_attns, ) - - -@add_start_docstrings( - "The OPT Model with a language modeling head. Can be used for summarization.", - OPT_START_DOCSTRING, -) -# Copied from transformers.models.bart.modeling_tf_bart.TFBartForConditionalGeneration with BART->OPT,Bart->OPT -class TFOPTForConditionalGeneration(TFOPTPretrainedModel, TFCausalLanguageModelingLoss): - _keys_to_ignore_on_load_unexpected = [ - r"model.encoder.embed_tokens.weight", - r"model.decoder.embed_tokens.weight", - ] - - _requires_load_weight_prefix = True - - def __init__(self, config, load_weight_prefix=None, *inputs, **kwargs): - super().__init__(config, *inputs, **kwargs) - self.model = TFOPTMainLayer(config, load_weight_prefix=load_weight_prefix, name="model") - self.use_cache = config.use_cache - # final_bias_logits is registered as a buffer in pytorch, so not trainable for the the sake of consistency. - self.final_logits_bias = self.add_weight( - name="final_logits_bias", shape=[1, config.vocab_size], initializer="zeros", trainable=False - ) - - def get_decoder(self): - return self.model.decoder - - def get_encoder(self): - return self.model.encoder - - def get_output_embeddings(self): - return self.get_input_embeddings() - - def set_output_embeddings(self, value): - self.set_input_embeddings(value) - - def get_bias(self): - return {"final_logits_bias": self.final_logits_bias} - - def set_bias(self, value): - self.final_logits_bias = value["final_logits_bias"] - - @add_start_docstrings_to_model_forward(OPT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=TFSeq2SeqLMOutput, config_class=_CONFIG_FOR_DOC) - @add_end_docstrings(OPT_GENERATION_EXAMPLE) - @unpack_inputs - def call( - self, - input_ids: Optional[TFModelInputType] = None, - attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, - decoder_input_ids: Optional[Union[np.ndarray, tf.Tensor]] = None, - decoder_attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, - head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, - decoder_head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, - cross_attn_head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, - encoder_outputs: Optional[TFBaseModelOutput] = None, - past_key_values: Optional[Tuple[Tuple[Union[np.ndarray, tf.Tensor]]]] = None, - inputs_embeds: Optional[Union[np.ndarray, tf.Tensor]] = None, - decoder_inputs_embeds: Optional[Union[np.ndarray, tf.Tensor]] = None, - use_cache: Optional[bool] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - labels: Optional[tf.Tensor] = None, - training: Optional[bool] = False, - ) -> Union[TFSeq2SeqLMOutput, Tuple[tf.Tensor]]: - r""" - labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., - config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored - (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. - - Returns: - - """ - - if labels is not None: - labels = tf.where( - labels == self.config.pad_token_id, - tf.cast(tf.fill(shape_list(labels), -100), labels.dtype), - labels, - ) - use_cache = False - if decoder_input_ids is None: - decoder_input_ids = shift_tokens_right( - labels, self.config.pad_token_id, self.config.decoder_start_token_id - ) - - outputs = self.model( - input_ids, - attention_mask=attention_mask, - decoder_input_ids=decoder_input_ids, - encoder_outputs=encoder_outputs, - decoder_attention_mask=decoder_attention_mask, - head_mask=head_mask, - decoder_head_mask=decoder_head_mask, - cross_attn_head_mask=cross_attn_head_mask, - past_key_values=past_key_values, - inputs_embeds=inputs_embeds, - decoder_inputs_embeds=decoder_inputs_embeds, - use_cache=use_cache, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - training=training, - ) - lm_logits = self.model.shared(outputs[0], mode="linear") - lm_logits = lm_logits + self.final_logits_bias - masked_lm_loss = None if labels is None else self.hf_compute_loss(labels, lm_logits) - - if not return_dict: - output = (lm_logits,) + outputs[1:] - return ((masked_lm_loss,) + output) if masked_lm_loss is not None else output - return TFSeq2SeqLMOutput( - loss=masked_lm_loss, - logits=lm_logits, - past_key_values=outputs.past_key_values, # index 1 of d outputs - decoder_hidden_states=outputs.decoder_hidden_states, # index 2 of d outputs - decoder_attentions=outputs.decoder_attentions, # index 3 of d outputs - cross_attentions=outputs.cross_attentions, # index 4 of d outputs - encoder_last_hidden_state=outputs.encoder_last_hidden_state, # index 0 of encoder outputs - encoder_hidden_states=outputs.encoder_hidden_states, # 1 of e out - encoder_attentions=outputs.encoder_attentions, # 2 of e out - ) - - def serving_output(self, output): - pkv = tf.tuple(output.past_key_values)[1] if self.config.use_cache else None - dec_hs = tf.convert_to_tensor(output.decoder_hidden_states) if self.config.output_hidden_states else None - dec_attns = tf.convert_to_tensor(output.decoder_attentions) if self.config.output_attentions else None - cross_attns = tf.convert_to_tensor(output.cross_attentions) if self.config.output_attentions else None - enc_hs = tf.convert_to_tensor(output.encoder_hidden_states) if self.config.output_hidden_states else None - enc_attns = tf.convert_to_tensor(output.encoder_attentions) if self.config.output_attentions else None - - return TFSeq2SeqLMOutput( - logits=output.logits, - past_key_values=pkv, - decoder_hidden_states=dec_hs, - decoder_attentions=dec_attns, - cross_attentions=cross_attns, - encoder_last_hidden_state=output.encoder_last_hidden_state, - encoder_hidden_states=enc_hs, - encoder_attentions=enc_attns, - ) - - def prepare_inputs_for_generation( - self, - decoder_input_ids, - past=None, - attention_mask=None, - head_mask=None, - decoder_head_mask=None, - cross_attn_head_mask=None, - use_cache=None, - encoder_outputs=None, - **kwargs - ): - # cut decoder_input_ids if past is used - if past is not None: - decoder_input_ids = decoder_input_ids[:, -1:] - - return { - "input_ids": None, # encoder_outputs is defined. input_ids not needed - "encoder_outputs": encoder_outputs, - "past_key_values": past, - "decoder_input_ids": decoder_input_ids, - "attention_mask": attention_mask, - "head_mask": head_mask, - "decoder_head_mask": decoder_head_mask, - "cross_attn_head_mask": cross_attn_head_mask, - "use_cache": use_cache, # change this to avoid caching (presumably for debugging) - } - - def prepare_decoder_input_ids_from_labels(self, labels: tf.Tensor): - return shift_tokens_right(labels, self.config.pad_token_id, self.config.decoder_start_token_id) - - @staticmethod - def _reorder_cache(past, beam_idx): - reordered_past = () - for layer_past in past: - # cached cross_attention states don't have to be reordered -> they are always the same - reordered_past += ( - tuple(tf.gather(past_state, beam_idx, axis=0) for past_state in layer_past[:2]) + layer_past[2:], - ) - return reordered_past From 14eeb1310a5eec12793aa066ed4f136e0d115a80 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Fri, 6 May 2022 13:34:59 +0200 Subject: [PATCH 032/145] removed useless dummy object for QA, SC and CG --- src/transformers/utils/dummy_flax_objects.py | 21 -------------------- src/transformers/utils/dummy_tf_objects.py | 7 ------- 2 files changed, 28 deletions(-) diff --git a/src/transformers/utils/dummy_flax_objects.py b/src/transformers/utils/dummy_flax_objects.py index e3723abacd841..e125683f6f5a5 100644 --- a/src/transformers/utils/dummy_flax_objects.py +++ b/src/transformers/utils/dummy_flax_objects.py @@ -809,27 +809,6 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["flax"]) -class FlaxOPTForConditionalGeneration(metaclass=DummyObject): - _backends = ["flax"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["flax"]) - - -class FlaxOPTForQuestionAnswering(metaclass=DummyObject): - _backends = ["flax"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["flax"]) - - -class FlaxOPTForSequenceClassification(metaclass=DummyObject): - _backends = ["flax"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["flax"]) - - class FlaxOPTModel(metaclass=DummyObject): _backends = ["flax"] diff --git a/src/transformers/utils/dummy_tf_objects.py b/src/transformers/utils/dummy_tf_objects.py index 4166a896f4907..242c6aeb1fa9f 100644 --- a/src/transformers/utils/dummy_tf_objects.py +++ b/src/transformers/utils/dummy_tf_objects.py @@ -1588,13 +1588,6 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["tf"]) -class TFOPTForConditionalGeneration(metaclass=DummyObject): - _backends = ["tf"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["tf"]) - - class TFOPTModel(metaclass=DummyObject): _backends = ["tf"] From 9c96f0977113676cfb673b7d2fe8bd9368c4b0ae Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Fri, 6 May 2022 13:35:41 +0200 Subject: [PATCH 033/145] removed tests for removed useless dummy object for QA, SC and CG --- tests/models/opt/test_modeling_flax_opt.py | 124 +-------------------- 1 file changed, 4 insertions(+), 120 deletions(-) diff --git a/tests/models/opt/test_modeling_flax_opt.py b/tests/models/opt/test_modeling_flax_opt.py index 665ca8db79122..25925b87f0709 100644 --- a/tests/models/opt/test_modeling_flax_opt.py +++ b/tests/models/opt/test_modeling_flax_opt.py @@ -16,7 +16,7 @@ import numpy as np import timeout_decorator # noqa -from transformers import GPT2Tokenizer, OPTConfig, is_flax_available +from transformers import OPTConfig, is_flax_available from transformers.testing_utils import require_flax, slow from ...generation.test_generation_flax_utils import FlaxGenerationTesterMixin @@ -33,13 +33,7 @@ import jax import jax.numpy as jnp - from transformers.models.opt.modeling_flax_opt import ( - FlaxOPTForConditionalGeneration, - FlaxOPTForQuestionAnswering, - FlaxOPTForSequenceClassification, - FlaxOPTModel, - shift_tokens_right, - ) + from transformers.models.opt.modeling_flax_opt import FlaxOPTModel, shift_tokens_right def prepare_opt_inputs_dict( @@ -270,48 +264,6 @@ def _get_config_and_data(self): ) return config, input_ids, batch_size - def test_sequence_classification_forward(self): - config, input_ids, batch_size = self._get_config_and_data() - model = FlaxOPTForSequenceClassification(config) - outputs = model(input_ids=input_ids, decoder_input_ids=input_ids) - expected_shape = (batch_size, config.num_labels) - self.assertEqual(outputs["logits"].shape, expected_shape) - - def test_question_answering_forward(self): - config, input_ids, batch_size = self._get_config_and_data() - model = FlaxOPTForQuestionAnswering(config) - outputs = model(input_ids=input_ids) - - self.assertEqual(outputs["start_logits"].shape, input_ids.shape) - self.assertEqual(outputs["end_logits"].shape, input_ids.shape) - - # @timeout_decorator.timeout(1) # not working with the decorator so far - def test_lm_forward(self): - config, input_ids, batch_size = self._get_config_and_data() - lm_model = FlaxOPTForConditionalGeneration(config) - outputs = lm_model(input_ids=input_ids) - expected_shape = (batch_size, input_ids.shape[1], config.vocab_size) - self.assertEqual(outputs["logits"].shape, expected_shape) - - def test_lm_uneven_forward(self): - config = OPTConfig( - vocab_size=self.vocab_size, - d_model=14, - encoder_layers=2, - decoder_layers=2, - encoder_attention_heads=2, - decoder_attention_heads=2, - encoder_ffn_dim=8, - decoder_ffn_dim=8, - max_position_embeddings=48, - ) - lm_model = FlaxOPTForConditionalGeneration(config) - context = np.array([[71, 82, 18, 33, 46, 91, 2], [68, 34, 26, 58, 30, 2, 1]], dtype=np.int64) - summary = np.array([[82, 71, 82, 18, 2], [58, 68, 2, 1, 1]], dtype=np.int64) - outputs = lm_model(input_ids=context, decoder_input_ids=summary) - expected_shape = (*summary.shape, config.vocab_size) - self.assertEqual(outputs["logits"].shape, expected_shape) - def test_shift_tokens_right(self): input_ids = np.array([[71, 82, 18, 33, 2, 1, 1], [68, 34, 26, 58, 30, 82, 2]], dtype=np.int64) shifted = shift_tokens_right(input_ids, 1, 2) @@ -325,17 +277,8 @@ def test_shift_tokens_right(self): @require_flax class FlaxOPTModelTest(FlaxModelTesterMixin, unittest.TestCase, FlaxGenerationTesterMixin): is_encoder_decoder = True - all_model_classes = ( - ( - FlaxOPTModel, - FlaxOPTForConditionalGeneration, - FlaxOPTForSequenceClassification, - FlaxOPTForQuestionAnswering, - ) - if is_flax_available() - else () - ) - all_generative_model_classes = (FlaxOPTForConditionalGeneration,) if is_flax_available() else () + all_model_classes = (FlaxOPTModel,) if is_flax_available() else () + all_generative_model_classes = () if is_flax_available() else () def setUp(self): self.model_tester = FlaxOPTModelTester(self) @@ -415,65 +358,6 @@ def test_model_from_pretrained(self): outputs = model(input_ids) self.assertIsNotNone(outputs) - @slow - def test_summarization_fast(self): - model = FlaxOPTForConditionalGeneration.from_pretrained("sshleifer/distilopt-cnn-6-6") - tokenizer = GPT2Tokenizer.from_pretrained("patrickvonplaten/opt_gpt2_tokenizer") - - input_str = "This sentence is made of three parts. Each part is important on its own. One part is about animals, the other part about planes, and the last part about housing." - - input_ids = tokenizer(input_str, return_tensors="np").input_ids - sequences = model.generate(input_ids, num_beams=2, max_length=20).sequences - - output_str = tokenizer.batch_decode(sequences)[0] - - assert ( - output_str == "This sentence is made of three parts. One part is about animals, the other part" - ) - - @slow - def test_cnn_summarization_same_as_fairseq(self): - model = FlaxOPTForConditionalGeneration.from_pretrained("facebook/opt-large-cnn") - tokenizer = GPT2Tokenizer.from_pretrained("patrickvonplaten/opt_gpt2_tokenizer") - - FRANCE_ARTICLE = ' Marseille, France (CNN)The French prosecutor leading an investigation into the crash of Germanwings Flight 9525 insisted Wednesday that he was not aware of any video footage from on board the plane. Marseille prosecutor Brice Robin told CNN that "so far no videos were used in the crash investigation." He added, "A person who has such a video needs to immediately give it to the investigators." Robin\'s comments follow claims by two magazines, German daily Bild and French Paris Match, of a cell phone video showing the harrowing final seconds from on board Germanwings Flight 9525 as it crashed into the French Alps. All 150 on board were killed. Paris Match and Bild reported that the video was recovered from a phone at the wreckage site. The two publications described the supposed video, but did not post it on their websites. The publications said that they watched the video, which was found by a source close to the investigation. "One can hear cries of \'My God\' in several languages," Paris Match reported. "Metallic banging can also be heard more than three times, perhaps of the pilot trying to open the cockpit door with a heavy object. Towards the end, after a heavy shake, stronger than the others, the screaming intensifies. Then nothing." "It is a very disturbing scene," said Julian Reichelt, editor-in-chief of Bild online. An official with France\'s accident investigation agency, the BEA, said the agency is not aware of any such video. Lt. Col. Jean-Marc Menichini, a French Gendarmerie spokesman in charge of communications on rescue efforts around the Germanwings crash site, told CNN that the reports were "completely wrong" and "unwarranted." Cell phones have been collected at the site, he said, but that they "hadn\'t been exploited yet." Menichini said he believed the cell phones would need to be sent to the Criminal Research Institute in Rosny sous-Bois, near Paris, in order to be analyzed by specialized technicians working hand-in-hand with investigators. But none of the cell phones found so far have been sent to the institute, Menichini said. Asked whether staff involved in the search could have leaked a memory card to the media, Menichini answered with a categorical "no." Reichelt told "Erin Burnett: Outfront" that he had watched the video and stood by the report, saying Bild and Paris Match are "very confident" that the clip is real. He noted that investigators only revealed they\'d recovered cell phones from the crash site after Bild and Paris Match published their reports. "That is something we did not know before. ... Overall we can say many things of the investigation weren\'t revealed by the investigation at the beginning," he said. What was mental state of Germanwings co-pilot? German airline Lufthansa confirmed Tuesday that co-pilot Andreas Lubitz had battled depression years before he took the controls of Germanwings Flight 9525, which he\'s accused of deliberately crashing last week in the French Alps. Lubitz told his Lufthansa flight training school in 2009 that he had a "previous episode of severe depression," the airline said Tuesday. Email correspondence between Lubitz and the school discovered in an internal investigation, Lufthansa said, included medical documents he submitted in connection with resuming his flight training. The announcement indicates that Lufthansa, the parent company of Germanwings, knew of Lubitz\'s battle with depression, allowed him to continue training and ultimately put him in the cockpit. Lufthansa, whose CEO Carsten Spohr previously said Lubitz was 100% fit to fly, described its statement Tuesday as a "swift and seamless clarification" and said it was sharing the information and documents -- including training and medical records -- with public prosecutors. Spohr traveled to the crash site Wednesday, where recovery teams have been working for the past week to recover human remains and plane debris scattered across a steep mountainside. He saw the crisis center set up in Seyne-les-Alpes, laid a wreath in the village of Le Vernet, closer to the crash site, where grieving families have left flowers at a simple stone memorial. Menichini told CNN late Tuesday that no visible human remains were left at the site but recovery teams would keep searching. French President Francois Hollande, speaking Tuesday, said that it should be possible to identify all the victims using DNA analysis by the end of the week, sooner than authorities had previously suggested. In the meantime, the recovery of the victims\' personal belongings will start Wednesday, Menichini said. Among those personal belongings could be more cell phones belonging to the 144 passengers and six crew on board. Check out the latest from our correspondents . The details about Lubitz\'s correspondence with the flight school during his training were among several developments as investigators continued to delve into what caused the crash and Lubitz\'s possible motive for downing the jet. A Lufthansa spokesperson told CNN on Tuesday that Lubitz had a valid medical certificate, had passed all his examinations and "held all the licenses required." Earlier, a spokesman for the prosecutor\'s office in Dusseldorf, Christoph Kumpa, said medical records reveal Lubitz suffered from suicidal tendencies at some point before his aviation career and underwent psychotherapy before he got his pilot\'s license. Kumpa emphasized there\'s no evidence suggesting Lubitz was suicidal or acting aggressively before the crash. Investigators are looking into whether Lubitz feared his medical condition would cause him to lose his pilot\'s license, a European government official briefed on the investigation told CNN on Tuesday. While flying was "a big part of his life," the source said, it\'s only one theory being considered. Another source, a law enforcement official briefed on the investigation, also told CNN that authorities believe the primary motive for Lubitz to bring down the plane was that he feared he would not be allowed to fly because of his medical problems. Lubitz\'s girlfriend told investigators he had seen an eye doctor and a neuropsychologist, both of whom deemed him unfit to work recently and concluded he had psychological issues, the European government official said. But no matter what details emerge about his previous mental health struggles, there\'s more to the story, said Brian Russell, a forensic psychologist. "Psychology can explain why somebody would turn rage inward on themselves about the fact that maybe they weren\'t going to keep doing their job and they\'re upset about that and so they\'re suicidal," he said. "But there is no mental illness that explains why somebody then feels entitled to also take that rage and turn it outward on 149 other people who had nothing to do with the person\'s problems." Germanwings crash compensation: What we know . Who was the captain of Germanwings Flight 9525? CNN\'s Margot Haddad reported from Marseille and Pamela Brown from Dusseldorf, while Laura Smith-Spark wrote from London. CNN\'s Frederik Pleitgen, Pamela Boykoff, Antonia Mortensen, Sandrine Amiel and Anna-Maja Rappard contributed to this report.' # @noq - - SHORTER_ARTICLE = ' (CNN)The Palestinian Authority officially became the 123rd member of the International Criminal Court on Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian territories. The formal accession was marked with a ceremony at The Hague, in the Netherlands, where the court is based. The Palestinians signed the ICC\'s founding Rome Statute in January, when they also accepted its jurisdiction over alleged crimes committed "in the occupied Palestinian territory, including East Jerusalem, since June 13, 2014." Later that month, the ICC opened a preliminary examination into the situation in Palestinian territories, paving the way for possible war crimes investigations against Israelis. As members of the court, Palestinians may be subject to counter-charges as well. Israel and the United States, neither of which is an ICC member, opposed the Palestinians\' efforts to join the body. But Palestinian Foreign Minister Riad al-Malki, speaking at Wednesday\'s ceremony, said it was a move toward greater justice. "As Palestine formally becomes a State Party to the Rome Statute today, the world is also a step closer to ending a long era of impunity and injustice," he said, according to an ICC news release. "Indeed, today brings us closer to our shared goals of justice and peace." Judge Kuniko Ozaki, a vice president of the ICC, said acceding to the treaty was just the first step for the Palestinians. "As the Rome Statute today enters into force for the State of Palestine, Palestine acquires all the rights as well as responsibilities that come with being a State Party to the Statute. These are substantive commitments, which cannot be taken lightly," she said. Rights group Human Rights Watch welcomed the development. "Governments seeking to penalize Palestine for joining the ICC should immediately end their pressure, and countries that support universal acceptance of the court\'s treaty should speak out to welcome its membership," said Balkees Jarrah, international justice counsel for the group. "What\'s objectionable is the attempts to undermine international justice, not Palestine\'s decision to join a treaty to which over 100 countries around the world are members." In January, when the preliminary ICC examination was opened, Israeli Prime Minister Benjamin Netanyahu described it as an outrage, saying the court was overstepping its boundaries. The United States also said it "strongly" disagreed with the court\'s decision. "As we have said repeatedly, we do not believe that Palestine is a state and therefore we do not believe that it is eligible to join the ICC," the State Department said in a statement. It urged the warring sides to resolve their differences through direct negotiations. "We will continue to oppose actions against Israel at the ICC as counterproductive to the cause of peace," it said. But the ICC begs to differ with the definition of a state for its purposes and refers to the territories as "Palestine." While a preliminary examination is not a formal investigation, it allows the court to review evidence and determine whether to investigate suspects on both sides. Prosecutor Fatou Bensouda said her office would "conduct its analysis in full independence and impartiality." The war between Israel and Hamas militants in Gaza last summer left more than 2,000 people dead. The inquiry will include alleged war crimes committed since June. The International Criminal Court was set up in 2002 to prosecute genocide, crimes against humanity and war crimes. CNN\'s Vasco Cotovio, Kareem Khadder and Faith Karimi contributed to this report.' - - # The below article tests that we don't add any hypotheses outside of the top n_beams - IRAN_ARTICLE = " (CNN)The United States and its negotiating partners reached a very strong framework agreement with Iran in Lausanne, Switzerland, on Thursday that limits Iran's nuclear program in such a way as to effectively block it from building a nuclear weapon. Expect pushback anyway, if the recent past is any harbinger. Just last month, in an attempt to head off such an agreement, House Speaker John Boehner invited Israeli Prime Minister Benjamin Netanyahu to preemptively blast it before Congress, and 47 senators sent a letter to the Iranian leadership warning them away from a deal. The debate that has already begun since the announcement of the new framework will likely result in more heat than light. It will not be helped by the gathering swirl of dubious assumptions and doubtful assertions. Let us address some of these: . The most misleading assertion, despite universal rejection by experts, is that the negotiations' objective at the outset was the total elimination of any nuclear program in Iran. That is the position of Netanyahu and his acolytes in the U.S. Congress. But that is not and never was the objective. If it had been, there would have been no Iranian team at the negotiating table. Rather, the objective has always been to structure an agreement or series of agreements so that Iran could not covertly develop a nuclear arsenal before the United States and its allies could respond. The new framework has exceeded expectations in achieving that goal. It would reduce Iran's low-enriched uranium stockpile, cut by two-thirds its number of installed centrifuges and implement a rigorous inspection regime. Another dubious assumption of opponents is that the Iranian nuclear program is a covert weapons program. Despite sharp accusations by some in the United States and its allies, Iran denies having such a program, and U.S. intelligence contends that Iran has not yet made the decision to build a nuclear weapon. Iran's continued cooperation with International Atomic Energy Agency inspections is further evidence on this point, and we'll know even more about Iran's program in the coming months and years because of the deal. In fact, the inspections provisions that are part of this agreement are designed to protect against any covert action by the Iranians. What's more, the rhetoric of some members of Congress has implied that the negotiations have been between only the United States and Iran (i.e., the 47 senators' letter warning that a deal might be killed by Congress or a future president). This of course is not the case. The talks were between Iran and the five permanent members of the U.N. Security Council (United States, United Kingdom, France, China and Russia) plus Germany, dubbed the P5+1. While the United States has played a leading role in the effort, it negotiated the terms alongside its partners. If the agreement reached by the P5+1 is rejected by Congress, it could result in an unraveling of the sanctions on Iran and threaten NATO cohesion in other areas. Another questionable assertion is that this agreement contains a sunset clause, after which Iran will be free to do as it pleases. Again, this is not the case. Some of the restrictions on Iran's nuclear activities, such as uranium enrichment, will be eased or eliminated over time, as long as 15 years. But most importantly, the framework agreement includes Iran's ratification of the Additional Protocol, which allows IAEA inspectors expanded access to nuclear sites both declared and nondeclared. This provision will be permanent. It does not sunset. Thus, going forward, if Iran decides to enrich uranium to weapons-grade levels, monitors will be able to detect such a move in a matter of days and alert the U.N. Security Council. Many in Congress have said that the agreement should be a formal treaty requiring the Senate to \"advise and consent.\" But the issue is not suited for a treaty. Treaties impose equivalent obligations on all signatories. For example, the New START treaty limits Russia and the United States to 1,550 deployed strategic warheads. But any agreement with Iran will not be so balanced. The restrictions and obligations in the final framework agreement will be imposed almost exclusively on Iran. The P5+1 are obligated only to ease and eventually remove most but not all economic sanctions, which were imposed as leverage to gain this final deal. Finally some insist that any agreement must address Iranian missile programs, human rights violations or support for Hamas or Hezbollah. As important as these issues are, and they must indeed be addressed, they are unrelated to the most important aim of a nuclear deal: preventing a nuclear Iran. To include them in the negotiations would be a poison pill. This agreement should be judged on its merits and on how it affects the security of our negotiating partners and allies, including Israel. Those judgments should be fact-based, not based on questionable assertions or dubious assumptions." - - ARTICLE_SUBWAY = ' New York (CNN)When Liana Barrientos was 23 years old, she got married in Westchester County, New York. A year later, she got married again in Westchester County, but to a different man and without divorcing her first husband. Only 18 days after that marriage, she got hitched yet again. Then, Barrientos declared "I do" five more times, sometimes only within two weeks of each other. In 2010, she married once more, this time in the Bronx. In an application for a marriage license, she stated it was her "first and only" marriage. Barrientos, now 39, is facing two criminal counts of "offering a false instrument for filing in the first degree," referring to her false statements on the 2010 marriage license application, according to court documents. Prosecutors said the marriages were part of an immigration scam. On Friday, she pleaded not guilty at State Supreme Court in the Bronx, according to her attorney, Christopher Wright, who declined to comment further. After leaving court, Barrientos was arrested and charged with theft of service and criminal trespass for allegedly sneaking into the New York subway through an emergency exit, said Detective Annette Markowski, a police spokeswoman. In total, Barrientos has been married 10 times, with nine of her marriages occurring between 1999 and 2002. All occurred either in Westchester County, Long Island, New Jersey or the Bronx. She is believed to still be married to four men, and at one time, she was married to eight men at once, prosecutors say. Prosecutors said the immigration scam involved some of her husbands, who filed for permanent residence status shortly after the marriages. Any divorces happened only after such filings were approved. It was unclear whether any of the men will be prosecuted. The case was referred to the Bronx District Attorney\'s Office by Immigration and Customs Enforcement and the Department of Homeland Security\'s Investigation Division. Seven of the men are from so-called "red-flagged" countries, including Egypt, Turkey, Georgia, Pakistan and Mali. Her eighth husband, Rashid Rajput, was deported in 2006 to his native Pakistan after an investigation by the Joint Terrorism Task Force. If convicted, Barrientos faces up to four years in prison. Her next court appearance is scheduled for May 18.' - - dct = tokenizer.batch_encode_plus( - [FRANCE_ARTICLE, SHORTER_ARTICLE, IRAN_ARTICLE, ARTICLE_SUBWAY], - max_length=1024, - padding="max_length", - truncation_strategy="only_first", - truncation=True, - return_tensors="np", - ) - - self.assertEqual(1024, dct["input_ids"].shape[1]) - hypotheses_batch = model.generate( - input_ids=dct["input_ids"], - attention_mask=dct["attention_mask"], - num_beams=2, - ).sequences - assert (hypotheses_batch[:, 1] == 0).all().item() - - EXPECTED = [ - "A French prosecutor says he is not aware of any video footage from on board the plane. Two German magazines claim to have found a cell phone video showing the crash. The publications say they watched the video, which was found by a source close to the investigation. All 150 on board the Germanwings flight were killed.", - "Palestinian Authority becomes 123rd member of the International Criminal Court. The move gives the court jurisdiction over alleged crimes in Palestinian territories. Israel and the United States opposed the Palestinians' efforts to join the body. But Palestinian Foreign Minister Riad al-Malki said it was a move toward greater justice.", - "U.S. and its negotiating partners reached a strong framework agreement with Iran. Peter Bergen: The debate that has already begun will likely result in more heat than light. Bergen: The most misleading assertion is that the negotiations' objective at the outset was the total elimination of any nuclear program.", - "Liana Barrientos, 39, has been married 10 times, sometimes within two weeks of each other. Prosecutors say the marriages were part of an immigration scam. She pleaded not guilty at State Supreme Court in the Bronx on Friday. If convicted, Barrientos faces up to four years in prison.", - ] - - generated_summaries = tokenizer.batch_decode( - hypotheses_batch.tolist(), clean_up_tokenization_spaces=True, skip_special_tokens=True - ) - assert generated_summaries == EXPECTED - class FlaxOPTStandaloneDecoderModelTester: def __init__( From 54fc96259e094903a720d102d97056b8f88ec8f7 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Fri, 6 May 2022 13:36:04 +0200 Subject: [PATCH 034/145] Removed head_mask using encoder layers which don't exist --- tests/models/opt/test_modeling_opt.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index 7a11ea944ce33..f97527debbe4d 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -51,8 +51,6 @@ def prepare_opt_inputs_dict( attention_mask = input_ids.ne(config.pad_token_id) if decoder_attention_mask is None: decoder_attention_mask = decoder_input_ids.ne(config.pad_token_id) - if head_mask is None: - head_mask = torch.ones(config.encoder_layers, config.encoder_attention_heads, device=torch_device) if decoder_head_mask is None: decoder_head_mask = torch.ones(config.decoder_layers, config.decoder_attention_heads, device=torch_device) if cross_attn_head_mask is None: @@ -815,6 +813,9 @@ def test_retain_grad_hidden_states_attentions(self): return +# TODO solve the @torch.no_grad() issue + + @require_torch @require_tokenizers class OPTEmbeddingsTest(unittest.TestCase): From 06f42ca6c567a15e48d7fe80a8d23a498cb48f0f Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Fri, 6 May 2022 13:36:19 +0200 Subject: [PATCH 035/145] fixed test --- tests/models/opt/test_modeling_tf_opt.py | 162 +---------------------- 1 file changed, 5 insertions(+), 157 deletions(-) diff --git a/tests/models/opt/test_modeling_tf_opt.py b/tests/models/opt/test_modeling_tf_opt.py index b55506e6152bb..81945ebdfc972 100644 --- a/tests/models/opt/test_modeling_tf_opt.py +++ b/tests/models/opt/test_modeling_tf_opt.py @@ -15,11 +15,8 @@ import unittest -import numpy as np - -from transformers import GPT2Tokenizer, OPTConfig, is_tf_available -from transformers.testing_utils import require_tf, slow -from transformers.utils import cached_property +from transformers import OPTConfig, is_tf_available +from transformers.testing_utils import require_tf from ...test_configuration_common import ConfigTester from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor @@ -29,7 +26,7 @@ if is_tf_available(): import tensorflow as tf - from transformers import TFOPTForConditionalGeneration, TFOPTModel + from transformers import TFOPTModel @require_tf @@ -178,8 +175,8 @@ def prepare_opt_inputs_dict( @require_tf class TFOPTModelTest(TFModelTesterMixin, TFCoreModelTesterMixin, unittest.TestCase): - all_model_classes = (TFOPTForConditionalGeneration, TFOPTModel) if is_tf_available() else () - all_generative_model_classes = (TFOPTForConditionalGeneration,) if is_tf_available() else () + all_model_classes = (TFOPTModel) if is_tf_available() else () + all_generative_model_classes = () if is_tf_available() else () is_encoder_decoder = True test_pruning = False test_onnx = True @@ -326,152 +323,3 @@ def _get_config_and_data(self): decoder_start_token_id=2, ) return config, input_ids, batch_size - - def test_lm_forward(self): - config, input_ids, batch_size = self._get_config_and_data() - decoder_lm_labels = ids_tensor([batch_size, input_ids.shape[1]], self.vocab_size) - lm_model = TFOPTForConditionalGeneration(config) - outputs = lm_model(input_ids=input_ids, labels=decoder_lm_labels, decoder_input_ids=input_ids, use_cache=False) - expected_shape = (batch_size, input_ids.shape[1], config.vocab_size) - self.assertEqual(outputs.logits.shape, expected_shape) - - def test_lm_uneven_forward(self): - config = OPTConfig( - vocab_size=10, - d_model=24, - encoder_layers=2, - decoder_layers=2, - encoder_attention_heads=2, - decoder_attention_heads=2, - encoder_ffn_dim=32, - decoder_ffn_dim=32, - max_position_embeddings=48, - ) - lm_model = TFOPTForConditionalGeneration(config) - context = tf.fill((7, 2), 4) - summary = tf.fill((7, 7), 6) - outputs = lm_model(input_ids=context, decoder_input_ids=summary, use_cache=False) - expected_shape = (*summary.shape, config.vocab_size) - self.assertEqual(outputs.logits.shape, expected_shape) - - -@slow -@require_tf -class TFOPTModelIntegrationTest(unittest.TestCase): - def test_inference_no_head(self): - model = TFOPTForConditionalGeneration.from_pretrained("facebook/opt-large").model - - input_ids = _long_tensor([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]) - attention_mask = tf.cast(tf.math.not_equal(input_ids, model.config.pad_token_id), tf.int8) - output = model(input_ids=input_ids, attention_mask=attention_mask)[0] - expected_shape = (1, 11, 1024) - self.assertEqual(output.shape, expected_shape) - expected_slice = tf.convert_to_tensor( - [[0.7144, 0.8143, -1.2813], [0.7144, 0.8143, -1.2813], [-0.0467, 2.5911, -2.1845]], - ) - tf.debugging.assert_near(output[:, :3, :3], expected_slice, atol=1e-3) - - def test_cnn_summarization_same_as_fairseq_hard(self): - hf = TFOPTForConditionalGeneration.from_pretrained("facebook/opt-large-cnn") - tok = self.tok - - FRANCE_ARTICLE = ' Marseille, France (CNN)The French prosecutor leading an investigation into the crash of Germanwings Flight 9525 insisted Wednesday that he was not aware of any video footage from on board the plane. Marseille prosecutor Brice Robin told CNN that "so far no videos were used in the crash investigation." He added, "A person who has such a video needs to immediately give it to the investigators." Robin\'s comments follow claims by two magazines, German daily Bild and French Paris Match, of a cell phone video showing the harrowing final seconds from on board Germanwings Flight 9525 as it crashed into the French Alps. All 150 on board were killed. Paris Match and Bild reported that the video was recovered from a phone at the wreckage site. The two publications described the supposed video, but did not post it on their websites. The publications said that they watched the video, which was found by a source close to the investigation. "One can hear cries of \'My God\' in several languages," Paris Match reported. "Metallic banging can also be heard more than three times, perhaps of the pilot trying to open the cockpit door with a heavy object. Towards the end, after a heavy shake, stronger than the others, the screaming intensifies. Then nothing." "It is a very disturbing scene," said Julian Reichelt, editor-in-chief of Bild online. An official with France\'s accident investigation agency, the BEA, said the agency is not aware of any such video. Lt. Col. Jean-Marc Menichini, a French Gendarmerie spokesman in charge of communications on rescue efforts around the Germanwings crash site, told CNN that the reports were "completely wrong" and "unwarranted." Cell phones have been collected at the site, he said, but that they "hadn\'t been exploited yet." Menichini said he believed the cell phones would need to be sent to the Criminal Research Institute in Rosny sous-Bois, near Paris, in order to be analyzed by specialized technicians working hand-in-hand with investigators. But none of the cell phones found so far have been sent to the institute, Menichini said. Asked whether staff involved in the search could have leaked a memory card to the media, Menichini answered with a categorical "no." Reichelt told "Erin Burnett: Outfront" that he had watched the video and stood by the report, saying Bild and Paris Match are "very confident" that the clip is real. He noted that investigators only revealed they\'d recovered cell phones from the crash site after Bild and Paris Match published their reports. "That is something we did not know before. ... Overall we can say many things of the investigation weren\'t revealed by the investigation at the beginning," he said. What was mental state of Germanwings co-pilot? German airline Lufthansa confirmed Tuesday that co-pilot Andreas Lubitz had battled depression years before he took the controls of Germanwings Flight 9525, which he\'s accused of deliberately crashing last week in the French Alps. Lubitz told his Lufthansa flight training school in 2009 that he had a "previous episode of severe depression," the airline said Tuesday. Email correspondence between Lubitz and the school discovered in an internal investigation, Lufthansa said, included medical documents he submitted in connection with resuming his flight training. The announcement indicates that Lufthansa, the parent company of Germanwings, knew of Lubitz\'s battle with depression, allowed him to continue training and ultimately put him in the cockpit. Lufthansa, whose CEO Carsten Spohr previously said Lubitz was 100% fit to fly, described its statement Tuesday as a "swift and seamless clarification" and said it was sharing the information and documents -- including training and medical records -- with public prosecutors. Spohr traveled to the crash site Wednesday, where recovery teams have been working for the past week to recover human remains and plane debris scattered across a steep mountainside. He saw the crisis center set up in Seyne-les-Alpes, laid a wreath in the village of Le Vernet, closer to the crash site, where grieving families have left flowers at a simple stone memorial. Menichini told CNN late Tuesday that no visible human remains were left at the site but recovery teams would keep searching. French President Francois Hollande, speaking Tuesday, said that it should be possible to identify all the victims using DNA analysis by the end of the week, sooner than authorities had previously suggested. In the meantime, the recovery of the victims\' personal belongings will start Wednesday, Menichini said. Among those personal belongings could be more cell phones belonging to the 144 passengers and six crew on board. Check out the latest from our correspondents . The details about Lubitz\'s correspondence with the flight school during his training were among several developments as investigators continued to delve into what caused the crash and Lubitz\'s possible motive for downing the jet. A Lufthansa spokesperson told CNN on Tuesday that Lubitz had a valid medical certificate, had passed all his examinations and "held all the licenses required." Earlier, a spokesman for the prosecutor\'s office in Dusseldorf, Christoph Kumpa, said medical records reveal Lubitz suffered from suicidal tendencies at some point before his aviation career and underwent psychotherapy before he got his pilot\'s license. Kumpa emphasized there\'s no evidence suggesting Lubitz was suicidal or acting aggressively before the crash. Investigators are looking into whether Lubitz feared his medical condition would cause him to lose his pilot\'s license, a European government official briefed on the investigation told CNN on Tuesday. While flying was "a big part of his life," the source said, it\'s only one theory being considered. Another source, a law enforcement official briefed on the investigation, also told CNN that authorities believe the primary motive for Lubitz to bring down the plane was that he feared he would not be allowed to fly because of his medical problems. Lubitz\'s girlfriend told investigators he had seen an eye doctor and a neuropsychologist, both of whom deemed him unfit to work recently and concluded he had psychological issues, the European government official said. But no matter what details emerge about his previous mental health struggles, there\'s more to the story, said Brian Russell, a forensic psychologist. "Psychology can explain why somebody would turn rage inward on themselves about the fact that maybe they weren\'t going to keep doing their job and they\'re upset about that and so they\'re suicidal," he said. "But there is no mental illness that explains why somebody then feels entitled to also take that rage and turn it outward on 149 other people who had nothing to do with the person\'s problems." Germanwings crash compensation: What we know . Who was the captain of Germanwings Flight 9525? CNN\'s Margot Haddad reported from Marseille and Pamela Brown from Dusseldorf, while Laura Smith-Spark wrote from London. CNN\'s Frederik Pleitgen, Pamela Boykoff, Antonia Mortensen, Sandrine Amiel and Anna-Maja Rappard contributed to this report.' # @noqa - EXPECTED_SUMMARY_FRANCE = 'French prosecutor says he\'s not aware of any video footage from on board the plane. German daily Bild and French Paris Match claim to have found a cell phone video of the crash. A French Gendarmerie spokesman calls the reports "completely wrong" and "unwarranted" German airline Lufthansa confirms co-pilot Andreas Lubitz had battled depression.' - - SHORTER_ARTICLE = ' (CNN)The Palestinian Authority officially became the 123rd member of the International Criminal Court on Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian territories. The formal accession was marked with a ceremony at The Hague, in the Netherlands, where the court is based. The Palestinians signed the ICC\'s founding Rome Statute in January, when they also accepted its jurisdiction over alleged crimes committed "in the occupied Palestinian territory, including East Jerusalem, since June 13, 2014." Later that month, the ICC opened a preliminary examination into the situation in Palestinian territories, paving the way for possible war crimes investigations against Israelis. As members of the court, Palestinians may be subject to counter-charges as well. Israel and the United States, neither of which is an ICC member, opposed the Palestinians\' efforts to join the body. But Palestinian Foreign Minister Riad al-Malki, speaking at Wednesday\'s ceremony, said it was a move toward greater justice. "As Palestine formally becomes a State Party to the Rome Statute today, the world is also a step closer to ending a long era of impunity and injustice," he said, according to an ICC news release. "Indeed, today brings us closer to our shared goals of justice and peace." Judge Kuniko Ozaki, a vice president of the ICC, said acceding to the treaty was just the first step for the Palestinians. "As the Rome Statute today enters into force for the State of Palestine, Palestine acquires all the rights as well as responsibilities that come with being a State Party to the Statute. These are substantive commitments, which cannot be taken lightly," she said. Rights group Human Rights Watch welcomed the development. "Governments seeking to penalize Palestine for joining the ICC should immediately end their pressure, and countries that support universal acceptance of the court\'s treaty should speak out to welcome its membership," said Balkees Jarrah, international justice counsel for the group. "What\'s objectionable is the attempts to undermine international justice, not Palestine\'s decision to join a treaty to which over 100 countries around the world are members." In January, when the preliminary ICC examination was opened, Israeli Prime Minister Benjamin Netanyahu described it as an outrage, saying the court was overstepping its boundaries. The United States also said it "strongly" disagreed with the court\'s decision. "As we have said repeatedly, we do not believe that Palestine is a state and therefore we do not believe that it is eligible to join the ICC," the State Department said in a statement. It urged the warring sides to resolve their differences through direct negotiations. "We will continue to oppose actions against Israel at the ICC as counterproductive to the cause of peace," it said. But the ICC begs to differ with the definition of a state for its purposes and refers to the territories as "Palestine." While a preliminary examination is not a formal investigation, it allows the court to review evidence and determine whether to investigate suspects on both sides. Prosecutor Fatou Bensouda said her office would "conduct its analysis in full independence and impartiality." The war between Israel and Hamas militants in Gaza last summer left more than 2,000 people dead. The inquiry will include alleged war crimes committed since June. The International Criminal Court was set up in 2002 to prosecute genocide, crimes against humanity and war crimes. CNN\'s Vasco Cotovio, Kareem Khadder and Faith Karimi contributed to this report.' - EXPECTED_SUMMARY_SHORTER = "The Palestinian Authority becomes the 123rd member of the International Criminal Court. The move gives the court jurisdiction over alleged crimes in Palestinian territories. Israel and the United States opposed the Palestinians' efforts to join the body. But Palestinian Foreign Minister Riad al-Malki said it was a move toward greater justice." - - # The below article tests that we don't add any hypotheses outside of the top n_beams - IRAN_ARTICLE = " (CNN)The United States and its negotiating partners reached a very strong framework agreement with Iran in Lausanne, Switzerland, on Thursday that limits Iran's nuclear program in such a way as to effectively block it from building a nuclear weapon. Expect pushback anyway, if the recent past is any harbinger. Just last month, in an attempt to head off such an agreement, House Speaker John Boehner invited Israeli Prime Minister Benjamin Netanyahu to preemptively blast it before Congress, and 47 senators sent a letter to the Iranian leadership warning them away from a deal. The debate that has already begun since the announcement of the new framework will likely result in more heat than light. It will not be helped by the gathering swirl of dubious assumptions and doubtful assertions. Let us address some of these: . The most misleading assertion, despite universal rejection by experts, is that the negotiations' objective at the outset was the total elimination of any nuclear program in Iran. That is the position of Netanyahu and his acolytes in the U.S. Congress. But that is not and never was the objective. If it had been, there would have been no Iranian team at the negotiating table. Rather, the objective has always been to structure an agreement or series of agreements so that Iran could not covertly develop a nuclear arsenal before the United States and its allies could respond. The new framework has exceeded expectations in achieving that goal. It would reduce Iran's low-enriched uranium stockpile, cut by two-thirds its number of installed centrifuges and implement a rigorous inspection regime. Another dubious assumption of opponents is that the Iranian nuclear program is a covert weapons program. Despite sharp accusations by some in the United States and its allies, Iran denies having such a program, and U.S. intelligence contends that Iran has not yet made the decision to build a nuclear weapon. Iran's continued cooperation with International Atomic Energy Agency inspections is further evidence on this point, and we'll know even more about Iran's program in the coming months and years because of the deal. In fact, the inspections provisions that are part of this agreement are designed to protect against any covert action by the Iranians. What's more, the rhetoric of some members of Congress has implied that the negotiations have been between only the United States and Iran (i.e., the 47 senators' letter warning that a deal might be killed by Congress or a future president). This of course is not the case. The talks were between Iran and the five permanent members of the U.N. Security Council (United States, United Kingdom, France, China and Russia) plus Germany, dubbed the P5+1. While the United States has played a leading role in the effort, it negotiated the terms alongside its partners. If the agreement reached by the P5+1 is rejected by Congress, it could result in an unraveling of the sanctions on Iran and threaten NATO cohesion in other areas. Another questionable assertion is that this agreement contains a sunset clause, after which Iran will be free to do as it pleases. Again, this is not the case. Some of the restrictions on Iran's nuclear activities, such as uranium enrichment, will be eased or eliminated over time, as long as 15 years. But most importantly, the framework agreement includes Iran's ratification of the Additional Protocol, which allows IAEA inspectors expanded access to nuclear sites both declared and nondeclared. This provision will be permanent. It does not sunset. Thus, going forward, if Iran decides to enrich uranium to weapons-grade levels, monitors will be able to detect such a move in a matter of days and alert the U.N. Security Council. Many in Congress have said that the agreement should be a formal treaty requiring the Senate to \"advise and consent.\" But the issue is not suited for a treaty. Treaties impose equivalent obligations on all signatories. For example, the New START treaty limits Russia and the United States to 1,550 deployed strategic warheads. But any agreement with Iran will not be so balanced. The restrictions and obligations in the final framework agreement will be imposed almost exclusively on Iran. The P5+1 are obligated only to ease and eventually remove most but not all economic sanctions, which were imposed as leverage to gain this final deal. Finally some insist that any agreement must address Iranian missile programs, human rights violations or support for Hamas or Hezbollah. As important as these issues are, and they must indeed be addressed, they are unrelated to the most important aim of a nuclear deal: preventing a nuclear Iran. To include them in the negotiations would be a poison pill. This agreement should be judged on its merits and on how it affects the security of our negotiating partners and allies, including Israel. Those judgments should be fact-based, not based on questionable assertions or dubious assumptions." - EXPECTED_SUMMARY_IRAN = "The U.S. and its negotiating partners reached a very strong framework agreement with Iran. Peter Bergen: The debate that has already begun will likely result in more heat than light. He says the agreement limits Iran's nuclear program in such a way as to effectively block it from building a nuclear weapon. Bergen says the most important aim of a nuclear deal is preventing a nuclear Iran." - - ARTICLE_SUBWAY = ' New York (CNN)When Liana Barrientos was 23 years old, she got married in Westchester County, New York. A year later, she got married again in Westchester County, but to a different man and without divorcing her first husband. Only 18 days after that marriage, she got hitched yet again. Then, Barrientos declared "I do" five more times, sometimes only within two weeks of each other. In 2010, she married once more, this time in the Bronx. In an application for a marriage license, she stated it was her "first and only" marriage. Barrientos, now 39, is facing two criminal counts of "offering a false instrument for filing in the first degree," referring to her false statements on the 2010 marriage license application, according to court documents. Prosecutors said the marriages were part of an immigration scam. On Friday, she pleaded not guilty at State Supreme Court in the Bronx, according to her attorney, Christopher Wright, who declined to comment further. After leaving court, Barrientos was arrested and charged with theft of service and criminal trespass for allegedly sneaking into the New York subway through an emergency exit, said Detective Annette Markowski, a police spokeswoman. In total, Barrientos has been married 10 times, with nine of her marriages occurring between 1999 and 2002. All occurred either in Westchester County, Long Island, New Jersey or the Bronx. She is believed to still be married to four men, and at one time, she was married to eight men at once, prosecutors say. Prosecutors said the immigration scam involved some of her husbands, who filed for permanent residence status shortly after the marriages. Any divorces happened only after such filings were approved. It was unclear whether any of the men will be prosecuted. The case was referred to the Bronx District Attorney\'s Office by Immigration and Customs Enforcement and the Department of Homeland Security\'s Investigation Division. Seven of the men are from so-called "red-flagged" countries, including Egypt, Turkey, Georgia, Pakistan and Mali. Her eighth husband, Rashid Rajput, was deported in 2006 to his native Pakistan after an investigation by the Joint Terrorism Task Force. If convicted, Barrientos faces up to four years in prison. Her next court appearance is scheduled for May 18.' - EXPECTED_SUMMARY_SUBWAY = "Liana Barrientos has been married 10 times, sometimes within two weeks of each other. Prosecutors say the marriages were part of an immigration scam. On Friday, she pleaded not guilty at State Supreme Court in the Bronx. She was arrested and charged with theft of service and criminal trespass for allegedly sneaking into the subway." - - dct = tok( - [FRANCE_ARTICLE, SHORTER_ARTICLE, IRAN_ARTICLE, ARTICLE_SUBWAY], - max_length=1024, - truncation_strategy="only_first", - padding="longest", - truncation=True, - return_tensors="tf", - ) - self.assertEqual(1024, dct["input_ids"].shape[1]) - hypotheses_batch = hf.generate( - input_ids=dct["input_ids"], - attention_mask=dct["attention_mask"], - ) - - assert hypotheses_batch[:, 1].numpy().tolist() == [0, 0, 0, 0] # test force_bos_token_to_be_generated - decoded = tok.batch_decode(hypotheses_batch, skip_special_tokens=True, clean_up_tokenization_spaces=False) - expected_batch = [ - EXPECTED_SUMMARY_FRANCE, - EXPECTED_SUMMARY_SHORTER, - EXPECTED_SUMMARY_IRAN, - EXPECTED_SUMMARY_SUBWAY, - ] - assert decoded == expected_batch - - @cached_property - def tok(self): - return GPT2Tokenizer.from_pretrained("patrickvonplaten/opt_gpt2_tokenizer") - - -@slow -@require_tf -class FasterTFOPTModelIntegrationTests(unittest.TestCase): - """These tests are useful for debugging since they operate on a model with 1 encoder layer and 1 decoder layer.""" - - @cached_property - def tok(self): - return GPT2Tokenizer.from_pretrained("patrickvonplaten/opt_gpt2_tokenizer") - - @cached_property - def xsum_1_1_model(self): - return TFOPTForConditionalGeneration.from_pretrained("sshleifer/distilopt-xsum-1-1") - - def test_xsum_1_1_generation(self): - model = self.xsum_1_1_model - assert model.model.decoder.embed_tokens._layer == model.model.shared - ARTICLE = 'The Palestinian Authority officially became the 123rd member of the International Criminal Court on Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian territories. The formal accession was marked with a ceremony at The Hague, in the Netherlands, where the court is based. The Palestinians signed the ICC\'s founding Rome Statute in January, when they also accepted its jurisdiction over alleged crimes committed "in the occupied Palestinian territory, including East Jerusalem, since June 13, 2014." Later that month, the ICC opened a preliminary examination into the situation in Palestinian territories, paving the way for possible war crimes investigations against Israelis. As members of the court, Palestinians may be subject to counter-charges as well. Israel and the United States, neither of which is an ICC member, opposed the Palestinians\' efforts to join the body. But Palestinian Foreign Minister Riad al-Malki, speaking at Wednesday\'s ceremony, said it was a move toward greater justice. "As Palestine formally becomes a State Party to the Rome Statute today, the world is also a step closer to ending a long era of impunity and injustice," he said, according to an ICC news release. "Indeed, today brings us closer to our shared goals of justice and peace." Judge Kuniko Ozaki, a vice president of the ICC, said acceding to the treaty was just the first step for the Palestinians. "As the Rome Statute today enters into force for the State of Palestine, Palestine acquires all the rights as well as responsibilities that come with being a State Party to the Statute. These are substantive commitments, which cannot be taken lightly," she said. Rights group Human Rights Watch welcomed the development. "Governments seeking to penalize Palestine for joining the ICC should immediately end their pressure, and countries that support universal acceptance of the court\'s treaty should speak out to welcome its membership," said Balkees Jarrah, international justice counsel for the group. "What\'s objectionable is the attempts to undermine international justice, not Palestine\'s decision to join a treaty to which over 100 countries around the world are members." In January, when the preliminary ICC examination was opened, Israeli Prime Minister Benjamin Netanyahu described it as an outrage, saying the court was overstepping its boundaries. The United States also said it "strongly" disagreed with the court\'s decision. "As we have said repeatedly, we do not believe that Palestine is a state and therefore we do not believe that it is eligible to join the ICC," the State Department said in a statement. It urged the warring sides to resolve their differences through direct negotiations. "We will continue to oppose actions against Israel at the ICC as counterproductive to the cause of peace," it said. But the ICC begs to differ with the definition of a state for its purposes and refers to the territories as "Palestine." While a preliminary examination is not a formal investigation, it allows the court to review evidence and determine whether to investigate suspects on both sides. Prosecutor Fatou Bensouda said her office would "conduct its analysis in full independence and impartiality." The war between Israel and Hamas militants in Gaza last summer left more than 2,000 people dead. The inquiry will include alleged war crimes committed since June. The International Criminal Court was set up in 2002 to prosecute genocide, crimes against humanity and war crimes.' - EXPECTED = " The International Criminal Court (ICC) has announced that it has been announced by the International Criminal court." - dct = self.tok(ARTICLE, return_tensors="tf") - generated_ids = model.generate(**dct, num_beams=4) - result = self.tok.batch_decode(generated_ids, skip_special_tokens=True)[0] - assert result == EXPECTED - - def test_xsum_1_1_batch_generation(self): - batch = self.tok( - [ - 'The Palestinian Authority officially became the 123rd member of the International Criminal Court on Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian territories. The formal accession was marked with a ceremony at The Hague, in the Netherlands, where the court is based. The Palestinians signed the ICC\'s founding Rome Statute in January, when they also accepted its jurisdiction over alleged crimes committed "in the occupied Palestinian territory, including East Jerusalem, since June 13, 2014." Later that month, the ICC opened a preliminary examination into the situation in Palestinian territories, paving the way for possible war crimes investigations against Israelis. As members of the court, Palestinians may be subject to counter-charges as well. Israel and the United States, neither of which is an ICC member, opposed the Palestinians\' efforts to join the body. But Palestinian Foreign Minister Riad al-Malki, speaking at Wednesday\'s ceremony, said it was a move toward greater justice. "As Palestine formally becomes a State Party to the Rome Statute today, the world is also a step closer to ending a long era of impunity and injustice," he said, according to an ICC news release. "Indeed, today brings us closer to our shared goals of justice and peace." Judge Kuniko Ozaki, a vice president of the ICC, said acceding to the treaty was just the first step for the Palestinians. "As the Rome Statute today enters into force for the State of Palestine, Palestine acquires all the rights as well as responsibilities that come with being a State Party to the Statute. These are substantive commitments, which cannot be taken lightly," she said. Rights group Human Rights Watch welcomed the development. "Governments seeking to penalize Palestine for joining the ICC should immediately end their pressure, and countries that support universal acceptance of the court\'s treaty should speak out to welcome its membership," said Balkees Jarrah, international justice counsel for the group. "What\'s objectionable is the attempts to undermine international justice, not Palestine\'s decision to join a treaty to which over 100 countries around the world are members." In January, when the preliminary ICC examination was opened, Israeli Prime Minister Benjamin Netanyahu described it as an outrage, saying the court was overstepping its boundaries. The United States also said it "strongly" disagreed with the court\'s decision. "As we have said repeatedly, we do not believe that Palestine is a state and therefore we do not believe that it is eligible to join the ICC," the State Department said in a statement. It urged the warring sides to resolve their differences through direct negotiations. "We will continue to oppose actions against Israel at the ICC as counterproductive to the cause of peace," it said. But the ICC begs to differ with the definition of a state for its purposes and refers to the territories as "Palestine." While a preliminary examination is not a formal investigation, it allows the court to review evidence and determine whether to investigate suspects on both sides. Prosecutor Fatou Bensouda said her office would "conduct its analysis in full independence and impartiality." The war between Israel and Hamas militants in Gaza last summer left more than 2,000 people dead. The inquiry will include alleged war crimes committed since June. The International Criminal Court was set up in 2002 to prosecute genocide, crimes against humanity and war crimes.', - 'The French prosecutor leading an investigation into the crash of Germanwings Flight 9525 insisted Wednesday that he was not aware of any video footage from on board the plane. Marseille prosecutor Brice Robin told CNN that "so far no videos were used in the crash investigation." He added, "A person who has such a video needs to immediately give it to the investigators." Robin\'s comments follow claims by two magazines, German daily Bild and French Paris Match, of a cell phone video showing the harrowing final seconds from on board Germanwings Flight 9525 as it crashed into the French Alps. All 150 on board were killed. Paris Match and Bild reported that the video was recovered from a phone at the wreckage site. The two publications described the supposed video, but did not post it on their websites. The publications said that they watched the video, which was found by a source close to the investigation. "One can hear cries of \'My God\' in several languages," Paris Match reported. "Metallic banging can also be heard more than three times, perhaps of the pilot trying to open the cockpit door with a heavy object. Towards the end, after a heavy shake, stronger than the others, the screaming intensifies. Then nothing." "It is a very disturbing scene," said Julian Reichelt, editor-in-chief of Bild online. An official with France\'s accident investigation agency, the BEA, said the agency is not aware of any such video. Lt. Col. Jean-Marc Menichini, a French Gendarmerie spokesman in charge of communications on rescue efforts around the Germanwings crash site, told CNN that the reports were "completely wrong" and "unwarranted." Cell phones have been collected at the site, he said, but that they "hadn\'t been exploited yet." Menichini said he believed the cell phones would need to be sent to the Criminal Research Institute in Rosny sous-Bois, near Paris, in order to be analyzed by specialized technicians working hand-in-hand with investigators. But none of the cell phones found so far have been sent to the institute, Menichini said. Asked whether staff involved in the search could have leaked a memory card to the media, Menichini answered with a categorical "no." Reichelt told "Erin Burnett: Outfront" that he had watched the video and stood by the report, saying Bild and Paris Match are "very confident" that the clip is real. He noted that investigators only revealed they\'d recovered cell phones from the crash site after Bild and Paris Match published their reports. "That is something we did not know before. ... Overall we can say many things of the investigation weren\'t revealed by the investigation at the beginning," he said. What was mental state of Germanwings co-pilot? German airline Lufthansa confirmed Tuesday that co-pilot Andreas Lubitz had battled depression years before he took the controls of Germanwings Flight 9525, which he\'s accused of deliberately crashing last week in the French Alps. Lubitz told his Lufthansa flight training school in 2009 that he had a "previous episode of severe depression," the airline said Tuesday. Email correspondence between Lubitz and the school discovered in an internal investigation, Lufthansa said, included medical documents he submitted in connection with resuming his flight training. The announcement indicates that Lufthansa, the parent company of Germanwings, knew of Lubitz\'s battle with depression, allowed him to continue training and ultimately put him in the cockpit. Lufthansa, whose CEO Carsten Spohr previously said Lubitz was 100% fit to fly, described its statement Tuesday as a "swift and seamless clarification" and said it was sharing the information and documents -- including training and medical records -- with public prosecutors. Spohr traveled to the crash site Wednesday, where recovery teams have been working for the past week to recover human remains and plane debris scattered across a steep mountainside. He saw the crisis center set up in Seyne-les-Alpes, laid a wreath in the village of Le Vernet, closer to the crash site, where grieving families have left flowers at a simple stone memorial. Menichini told CNN late Tuesday that no visible human remains were left at the site but recovery teams would keep searching. French President Francois Hollande, speaking Tuesday, said that it should be possible to identify all the victims using DNA analysis by the end of the week, sooner than authorities had previously suggested. In the meantime, the recovery of the victims\' personal belongings will start Wednesday, Menichini said. Among those personal belongings could be more cell phones belonging to the 144 passengers and six crew on board. Check out the latest from our correspondents . The details about Lubitz\'s correspondence with the flight school during his training were among several developments as investigators continued to delve into what caused the crash and Lubitz\'s possible motive for downing the jet. A Lufthansa spokesperson told CNN on Tuesday that Lubitz had a valid medical certificate, had passed all his examinations and "held all the licenses required." Earlier, a spokesman for the prosecutor\'s office in Dusseldorf, Christoph Kumpa, said medical records reveal Lubitz suffered from suicidal tendencies at some point before his aviation career and underwent psychotherapy before he got his pilot\'s license. Kumpa emphasized there\'s no evidence suggesting Lubitz was suicidal or acting aggressively before the crash. Investigators are looking into whether Lubitz feared his medical condition would cause him to lose his pilot\'s license, a European government official briefed on the investigation told CNN on Tuesday. While flying was "a big part of his life," the source said, it\'s only one theory being considered. Another source, a law enforcement official briefed on the investigation, also told CNN that authorities believe the primary motive for Lubitz to bring down the plane was that he feared he would not be allowed to fly because of his medical problems. Lubitz\'s girlfriend told investigators he had seen an eye doctor and a neuropsychologist, both of whom deemed him unfit to work recently and concluded he had psychological issues, the European government official said. But no matter what details emerge about his previous mental health struggles, there\'s more to the story, said Brian Russell, a forensic psychologist. "Psychology can explain why somebody would turn rage inward on themselves about the fact that maybe they weren\'t going to keep doing their job and they\'re upset about that and so they\'re suicidal," he said. "But there is no mental illness that explains why somebody then feels entitled to also take that rage and turn it outward on 149 other people who had nothing to do with the person\'s problems." Germanwings crash compensation: What we know . Who was the captain of Germanwings Flight 9525? CNN\'s Margot Haddad reported from Marseille and Pamela Brown from Dusseldorf, while Laura Smith-Spark wrote from London. CNN\'s Frederik Pleitgen, Pamela Boykoff, Antonia Mortensen, Sandrine Amiel and Anna-Maja Rappard contributed to this report.', - ], - return_tensors="tf", - padding="longest", - truncation=True, - ) - generated_ids = self.xsum_1_1_model.generate(**batch, num_beams=4) - result = self.tok.batch_decode(generated_ids, skip_special_tokens=True) - assert ( - result[0] - == " The International Criminal Court (ICC) has announced that it has been announced by the International Criminal court." - ) - assert ( - result[1] - == " An investigation into the crash that killed at least 10 people in the French capital has been released by the French police investigating the crash." - ) - - def test_encoder_equiv(self): - batch = self.tok( - [ - 'The Palestinian Authority officially became the 123rd member of the International Criminal Court on Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian territories. The formal accession was marked with a ceremony at The Hague, in the Netherlands, where the court is based. The Palestinians signed the ICC\'s founding Rome Statute in January, when they also accepted its jurisdiction over alleged crimes committed "in the occupied Palestinian territory, including East Jerusalem, since June 13, 2014." Later that month, the ICC opened a preliminary examination into the situation in Palestinian territories, paving the way for possible war crimes investigations against Israelis. As members of the court, Palestinians may be subject to counter-charges as well. Israel and the United States, neither of which is an ICC member, opposed the Palestinians\' efforts to join the body. But Palestinian Foreign Minister Riad al-Malki, speaking at Wednesday\'s ceremony, said it was a move toward greater justice. "As Palestine formally becomes a State Party to the Rome Statute today, the world is also a step closer to ending a long era of impunity and injustice," he said, according to an ICC news release. "Indeed, today brings us closer to our shared goals of justice and peace." Judge Kuniko Ozaki, a vice president of the ICC, said acceding to the treaty was just the first step for the Palestinians. "As the Rome Statute today enters into force for the State of Palestine, Palestine acquires all the rights as well as responsibilities that come with being a State Party to the Statute. These are substantive commitments, which cannot be taken lightly," she said. Rights group Human Rights Watch welcomed the development. "Governments seeking to penalize Palestine for joining the ICC should immediately end their pressure, and countries that support universal acceptance of the court\'s treaty should speak out to welcome its membership," said Balkees Jarrah, international justice counsel for the group. "What\'s objectionable is the attempts to undermine international justice, not Palestine\'s decision to join a treaty to which over 100 countries around the world are members." In January, when the preliminary ICC examination was opened, Israeli Prime Minister Benjamin Netanyahu described it as an outrage, saying the court was overstepping its boundaries. The United States also said it "strongly" disagreed with the court\'s decision. "As we have said repeatedly, we do not believe that Palestine is a state and therefore we do not believe that it is eligible to join the ICC," the State Department said in a statement. It urged the warring sides to resolve their differences through direct negotiations. "We will continue to oppose actions against Israel at the ICC as counterproductive to the cause of peace," it said. But the ICC begs to differ with the definition of a state for its purposes and refers to the territories as "Palestine." While a preliminary examination is not a formal investigation, it allows the court to review evidence and determine whether to investigate suspects on both sides. Prosecutor Fatou Bensouda said her office would "conduct its analysis in full independence and impartiality." The war between Israel and Hamas militants in Gaza last summer left more than 2,000 people dead. The inquiry will include alleged war crimes committed since June. The International Criminal Court was set up in 2002 to prosecute genocide, crimes against humanity and war crimes.', - 'The French prosecutor leading an investigation into the crash of Germanwings Flight 9525 insisted Wednesday that he was not aware of any video footage from on board the plane. Marseille prosecutor Brice Robin told CNN that "so far no videos were used in the crash investigation." He added, "A person who has such a video needs to immediately give it to the investigators." Robin\'s comments follow claims by two magazines, German daily Bild and French Paris Match, of a cell phone video showing the harrowing final seconds from on board Germanwings Flight 9525 as it crashed into the French Alps. All 150 on board were killed. Paris Match and Bild reported that the video was recovered from a phone at the wreckage site. The two publications described the supposed video, but did not post it on their websites. The publications said that they watched the video, which was found by a source close to the investigation. "One can hear cries of \'My God\' in several languages," Paris Match reported. "Metallic banging can also be heard more than three times, perhaps of the pilot trying to open the cockpit door with a heavy object. Towards the end, after a heavy shake, stronger than the others, the screaming intensifies. Then nothing." "It is a very disturbing scene," said Julian Reichelt, editor-in-chief of Bild online. An official with France\'s accident investigation agency, the BEA, said the agency is not aware of any such video. Lt. Col. Jean-Marc Menichini, a French Gendarmerie spokesman in charge of communications on rescue efforts around the Germanwings crash site, told CNN that the reports were "completely wrong" and "unwarranted." Cell phones have been collected at the site, he said, but that they "hadn\'t been exploited yet." Menichini said he believed the cell phones would need to be sent to the Criminal Research Institute in Rosny sous-Bois, near Paris, in order to be analyzed by specialized technicians working hand-in-hand with investigators. But none of the cell phones found so far have been sent to the institute, Menichini said. Asked whether staff involved in the search could have leaked a memory card to the media, Menichini answered with a categorical "no." Reichelt told "Erin Burnett: Outfront" that he had watched the video and stood by the report, saying Bild and Paris Match are "very confident" that the clip is real. He noted that investigators only revealed they\'d recovered cell phones from the crash site after Bild and Paris Match published their reports. "That is something we did not know before. ... Overall we can say many things of the investigation weren\'t revealed by the investigation at the beginning," he said. What was mental state of Germanwings co-pilot? German airline Lufthansa confirmed Tuesday that co-pilot Andreas Lubitz had battled depression years before he took the controls of Germanwings Flight 9525, which he\'s accused of deliberately crashing last week in the French Alps. Lubitz told his Lufthansa flight training school in 2009 that he had a "previous episode of severe depression," the airline said Tuesday. Email correspondence between Lubitz and the school discovered in an internal investigation, Lufthansa said, included medical documents he submitted in connection with resuming his flight training. The announcement indicates that Lufthansa, the parent company of Germanwings, knew of Lubitz\'s battle with depression, allowed him to continue training and ultimately put him in the cockpit. Lufthansa, whose CEO Carsten Spohr previously said Lubitz was 100% fit to fly, described its statement Tuesday as a "swift and seamless clarification" and said it was sharing the information and documents -- including training and medical records -- with public prosecutors. Spohr traveled to the crash site Wednesday, where recovery teams have been working for the past week to recover human remains and plane debris scattered across a steep mountainside. He saw the crisis center set up in Seyne-les-Alpes, laid a wreath in the village of Le Vernet, closer to the crash site, where grieving families have left flowers at a simple stone memorial. Menichini told CNN late Tuesday that no visible human remains were left at the site but recovery teams would keep searching. French President Francois Hollande, speaking Tuesday, said that it should be possible to identify all the victims using DNA analysis by the end of the week, sooner than authorities had previously suggested. In the meantime, the recovery of the victims\' personal belongings will start Wednesday, Menichini said. Among those personal belongings could be more cell phones belonging to the 144 passengers and six crew on board. Check out the latest from our correspondents . The details about Lubitz\'s correspondence with the flight school during his training were among several developments as investigators continued to delve into what caused the crash and Lubitz\'s possible motive for downing the jet. A Lufthansa spokesperson told CNN on Tuesday that Lubitz had a valid medical certificate, had passed all his examinations and "held all the licenses required." Earlier, a spokesman for the prosecutor\'s office in Dusseldorf, Christoph Kumpa, said medical records reveal Lubitz suffered from suicidal tendencies at some point before his aviation career and underwent psychotherapy before he got his pilot\'s license. Kumpa emphasized there\'s no evidence suggesting Lubitz was suicidal or acting aggressively before the crash. Investigators are looking into whether Lubitz feared his medical condition would cause him to lose his pilot\'s license, a European government official briefed on the investigation told CNN on Tuesday. While flying was "a big part of his life," the source said, it\'s only one theory being considered. Another source, a law enforcement official briefed on the investigation, also told CNN that authorities believe the primary motive for Lubitz to bring down the plane was that he feared he would not be allowed to fly because of his medical problems. Lubitz\'s girlfriend told investigators he had seen an eye doctor and a neuropsychologist, both of whom deemed him unfit to work recently and concluded he had psychological issues, the European government official said. But no matter what details emerge about his previous mental health struggles, there\'s more to the story, said Brian Russell, a forensic psychologist. "Psychology can explain why somebody would turn rage inward on themselves about the fact that maybe they weren\'t going to keep doing their job and they\'re upset about that and so they\'re suicidal," he said. "But there is no mental illness that explains why somebody then feels entitled to also take that rage and turn it outward on 149 other people who had nothing to do with the person\'s problems." Germanwings crash compensation: What we know . Who was the captain of Germanwings Flight 9525? CNN\'s Margot Haddad reported from Marseille and Pamela Brown from Dusseldorf, while Laura Smith-Spark wrote from London. CNN\'s Frederik Pleitgen, Pamela Boykoff, Antonia Mortensen, Sandrine Amiel and Anna-Maja Rappard contributed to this report.', - ], - return_tensors="tf", - padding="longest", - truncation=True, - ) - features = self.xsum_1_1_model.get_encoder()(**batch).last_hidden_state - - expected = np.array([[-0.0828, -0.0251, -0.0674], [0.1277, 0.3311, -0.0255], [0.2613, -0.0840, -0.2763]]) - assert np.allclose(features[0, :3, :3].numpy(), expected, atol=1e-3) From 76e52aca55100fd24a71a13f96dd453995339d13 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Fri, 6 May 2022 13:37:10 +0200 Subject: [PATCH 036/145] fix line --- src/transformers/models/opt/modeling_flax_opt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/models/opt/modeling_flax_opt.py b/src/transformers/models/opt/modeling_flax_opt.py index cd75183f071a2..2bc8412b5b257 100644 --- a/src/transformers/models/opt/modeling_flax_opt.py +++ b/src/transformers/models/opt/modeling_flax_opt.py @@ -1250,6 +1250,7 @@ class FlaxOPTModel(FlaxOPTPreTrainedModel): FlaxOPTModel, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSeq2SeqModelOutput, _CONFIG_FOR_DOC ) + # Copied from transformers.models.bart.modeling_flax_bart.FlaxBartDecoderPreTrainedModel with BART->OPT,Bart->OPT class FlaxOPTDecoderPreTrainedModel(FlaxPreTrainedModel): config_class = OPTConfig From 556c2f4b745a44b0fff9d5fc31cb03081fa54fb7 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Fri, 6 May 2022 14:32:23 +0200 Subject: [PATCH 037/145] added OPT to toctree --- docs/source/en/_toctree.yml | 2 ++ src/transformers/models/opt/modeling_opt.py | 11 +---------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index 6e9ef32c72f61..eae81cf7b3c0b 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -268,6 +268,8 @@ title: Nyströmformer - local: model_doc/openai-gpt title: OpenAI GPT + - local: model_doc/opt + title: OPT - local: model_doc/gpt2 title: OpenAI GPT2 - local: model_doc/gptj diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 8a0013f38d517..5669c2ad1bade 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -45,20 +45,11 @@ _CHECKPOINT_FOR_DOC = "" _CONFIG_FOR_DOC = "OPTConfig" -_TOKENIZER_FOR_DOC = "OPTTokenizer" +_TOKENIZER_FOR_DOC = "GPT2Tokenizer" # Base model docstring _EXPECTED_OUTPUT_SHAPE = [1, 8, 768] -# SequenceClassification docstring -_CHECKPOINT_FOR_SEQUENCE_CLASSIFICATION = "valhalla/opt-large-sst2" -_SEQ_CLASS_EXPECTED_LOSS = 0.0 -_SEQ_CLASS_EXPECTED_OUTPUT = "'POSITIVE'" - -# QuestionAsnwering docstring -_CHECKPOINT_FOR_QA = "valhalla/opt-large-finetuned-squadv1" -_QA_EXPECTED_LOSS = 0.59 -_QA_EXPECTED_OUTPUT = "' nice puppet'" OPT_PRETRAINED_MODEL_ARCHIVE_LIST = [ From 14600252a1317f224cb6d1f36f904ae7bad898ca Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Fri, 6 May 2022 15:50:42 +0200 Subject: [PATCH 038/145] Updated model path with pushed weigths --- src/transformers/models/opt/modeling_opt.py | 1 - tests/models/opt/test_modeling_opt.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 5669c2ad1bade..0a736092bcf97 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -51,7 +51,6 @@ _EXPECTED_OUTPUT_SHAPE = [1, 8, 768] - OPT_PRETRAINED_MODEL_ARCHIVE_LIST = [ "", # See all OPT models at https://huggingface.co/models?filter=opt diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index f97527debbe4d..82a66f43147d1 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -821,7 +821,7 @@ def test_retain_grad_hidden_states_attentions(self): class OPTEmbeddingsTest(unittest.TestCase): def setUp(self): super().setUp() - self.path_model = "/home/younes/Desktop/Work/data/opt-350-m/" # TODO add the model on the hub + self.path_model = "Arthurz/opt-350-m" # TODO add the model on the hub # self.path_logits_meta = "/home/younes/Desktop/Work/metaseq-conversion/logits_metaseq_gpt2_tokenizer.p" # TODO add the logits somewhere? # @unittest.skip("Skipping unless we find a solution to store the model") @@ -863,7 +863,7 @@ def test_logits(self): class OPTGenerationTest(unittest.TestCase): def setUp(self): super().setUp() - self.path_model = "/home/younes/Desktop/Work/data/opt-350-m/" # TODO add the model on the hub + self.path_model = "Arthurz/opt-350-m" # self.path_logits_meta = "/home/younes/Desktop/Work/metaseq-conversion/logits_metaseq_gpt2_tokenizer.p" # TODO add the logits somewhere? @unittest.skip("Skipping unless we find a solution to store the model") From db100a583a6c7b99042e0457dcf2c94cdc6daad4 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Fri, 6 May 2022 17:29:59 +0200 Subject: [PATCH 039/145] fix model path --- tests/models/opt/test_modeling_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index 82a66f43147d1..4419ecdbf26fd 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -821,7 +821,7 @@ def test_retain_grad_hidden_states_attentions(self): class OPTEmbeddingsTest(unittest.TestCase): def setUp(self): super().setUp() - self.path_model = "Arthurz/opt-350-m" # TODO add the model on the hub + self.path_model = "ArthurZ/opt-350m" # TODO add the model on the hub # self.path_logits_meta = "/home/younes/Desktop/Work/metaseq-conversion/logits_metaseq_gpt2_tokenizer.p" # TODO add the logits somewhere? # @unittest.skip("Skipping unless we find a solution to store the model") @@ -863,7 +863,7 @@ def test_logits(self): class OPTGenerationTest(unittest.TestCase): def setUp(self): super().setUp() - self.path_model = "Arthurz/opt-350-m" + self.path_model = "ArthurZ/opt-350m" # self.path_logits_meta = "/home/younes/Desktop/Work/metaseq-conversion/logits_metaseq_gpt2_tokenizer.p" # TODO add the logits somewhere? @unittest.skip("Skipping unless we find a solution to store the model") From c10f3476c7628c33cc70583a52811027deea30a3 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Fri, 6 May 2022 18:22:58 +0200 Subject: [PATCH 040/145] fixed code quality --- docs/source/en/model_doc/bert-generation.mdx | 4 ++-- docs/source/en/model_doc/luke.mdx | 2 +- .../modeling_encoder_decoder.py | 2 +- .../modeling_tf_encoder_decoder.py | 2 +- .../models/gpt2/modeling_tf_gpt2.py | 2 +- .../models/imagegpt/modeling_imagegpt.py | 2 +- .../models/longformer/modeling_longformer.py | 8 +++---- src/transformers/models/luke/modeling_luke.py | 6 ++--- .../models/openai/modeling_openai.py | 2 +- .../models/openai/modeling_tf_openai.py | 4 ++-- .../models/prophetnet/modeling_prophetnet.py | 6 ++--- src/transformers/models/rag/modeling_rag.py | 4 ++-- src/transformers/models/rag/retrieval_rag.py | 2 +- .../models/realm/modeling_realm.py | 2 +- .../modeling_tf_speech_to_text.py | 2 +- .../models/t5/modeling_flax_t5.py | 2 +- src/transformers/models/t5/modeling_t5.py | 6 ++--- src/transformers/models/t5/modeling_tf_t5.py | 6 ++--- .../models/tapas/modeling_tapas.py | 2 +- .../models/tapas/modeling_tf_tapas.py | 2 +- .../modeling_tf_vision_encoder_decoder.py | 2 +- .../models/wav2vec2/modeling_flax_wav2vec2.py | 4 ++-- src/transformers/models/xlm/modeling_xlm.py | 2 +- .../xlm_prophetnet/modeling_xlm_prophetnet.py | 4 ++-- .../models/xlnet/modeling_tf_xlnet.py | 8 +++---- .../models/xlnet/modeling_xlnet.py | 24 +++++++++---------- 26 files changed, 56 insertions(+), 56 deletions(-) diff --git a/docs/source/en/model_doc/bert-generation.mdx b/docs/source/en/model_doc/bert-generation.mdx index 3c6c229b6a593..e300917ea5e6f 100644 --- a/docs/source/en/model_doc/bert-generation.mdx +++ b/docs/source/en/model_doc/bert-generation.mdx @@ -49,7 +49,7 @@ Usage: >>> input_ids = tokenizer( ... "This is a long article to summarize", add_special_tokens=False, return_tensors="pt" ->>> ).input_ids +... ).input_ids >>> labels = tokenizer("This is a short summary", return_tensors="pt").input_ids >>> # train... @@ -67,7 +67,7 @@ Usage: >>> input_ids = tokenizer( ... "This is the first sentence. This is the second sentence.", add_special_tokens=False, return_tensors="pt" ->>> ).input_ids +... ).input_ids >>> outputs = sentence_fuser.generate(input_ids) diff --git a/docs/source/en/model_doc/luke.mdx b/docs/source/en/model_doc/luke.mdx index 1bbf6acb84544..6900367bb8369 100644 --- a/docs/source/en/model_doc/luke.mdx +++ b/docs/source/en/model_doc/luke.mdx @@ -97,7 +97,7 @@ Example: >>> entities = [ ... "Beyoncé", ... "Los Angeles", ->>> ] # Wikipedia entity titles corresponding to the entity mentions "Beyoncé" and "Los Angeles" +... ] # Wikipedia entity titles corresponding to the entity mentions "Beyoncé" and "Los Angeles" >>> entity_spans = [(0, 7), (17, 28)] # character-based entity spans corresponding to "Beyoncé" and "Los Angeles" >>> inputs = tokenizer(text, entities=entities, entity_spans=entity_spans, add_prefix_space=True, return_tensors="pt") >>> outputs = model(**inputs) diff --git a/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py b/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py index c9d4c00c668f1..a5e1b8311f9d3 100644 --- a/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py +++ b/src/transformers/models/encoder_decoder/modeling_encoder_decoder.py @@ -457,7 +457,7 @@ def forward( >>> tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") >>> model = EncoderDecoderModel.from_encoder_decoder_pretrained( ... "bert-base-uncased", "bert-base-uncased" - >>> ) # initialize Bert2Bert from pre-trained checkpoints + ... ) # initialize Bert2Bert from pre-trained checkpoints >>> # training >>> model.config.decoder_start_token_id = tokenizer.cls_token_id diff --git a/src/transformers/models/encoder_decoder/modeling_tf_encoder_decoder.py b/src/transformers/models/encoder_decoder/modeling_tf_encoder_decoder.py index 9e92e767b1b85..96c93d31cac25 100644 --- a/src/transformers/models/encoder_decoder/modeling_tf_encoder_decoder.py +++ b/src/transformers/models/encoder_decoder/modeling_tf_encoder_decoder.py @@ -528,7 +528,7 @@ def call( >>> # forward >>> input_ids = tokenizer.encode( ... "Hello, my dog is cute", add_special_tokens=True, return_tensors="tf" - >>> ) # Batch size 1 + ... ) # Batch size 1 >>> outputs = model(input_ids=input_ids, decoder_input_ids=input_ids) >>> # training diff --git a/src/transformers/models/gpt2/modeling_tf_gpt2.py b/src/transformers/models/gpt2/modeling_tf_gpt2.py index 3a11f46bdb3be..45d29b6779ee1 100644 --- a/src/transformers/models/gpt2/modeling_tf_gpt2.py +++ b/src/transformers/models/gpt2/modeling_tf_gpt2.py @@ -1061,7 +1061,7 @@ def call( >>> embedding_layer = model.resize_token_embeddings( ... len(tokenizer) - >>> ) # Update the model embeddings with the new vocabulary size + ... ) # Update the model embeddings with the new vocabulary size >>> choices = ["Hello, my dog is cute [CLS]", "Hello, my cat is cute [CLS]"] >>> encoded_choices = [tokenizer.encode(s) for s in choices] diff --git a/src/transformers/models/imagegpt/modeling_imagegpt.py b/src/transformers/models/imagegpt/modeling_imagegpt.py index 5866744bd8d0a..22186a6159e37 100755 --- a/src/transformers/models/imagegpt/modeling_imagegpt.py +++ b/src/transformers/models/imagegpt/modeling_imagegpt.py @@ -1000,7 +1000,7 @@ def forward( >>> samples = output[:, 1:].cpu().detach().numpy() >>> samples_img = [ ... np.reshape(np.rint(127.5 * (clusters[s] + 1.0)), [n_px, n_px, 3]).astype(np.uint8) for s in samples - >>> ] # convert color cluster tokens back to pixels + ... ] # convert color cluster tokens back to pixels >>> f, axes = plt.subplots(1, batch_size, dpi=300) >>> for img, ax in zip(samples_img, axes): diff --git a/src/transformers/models/longformer/modeling_longformer.py b/src/transformers/models/longformer/modeling_longformer.py index 32210ac65b60b..20a2e9d239e24 100755 --- a/src/transformers/models/longformer/modeling_longformer.py +++ b/src/transformers/models/longformer/modeling_longformer.py @@ -1634,10 +1634,10 @@ def forward( >>> attention_mask = torch.ones( ... input_ids.shape, dtype=torch.long, device=input_ids.device - >>> ) # initialize to local attention + ... ) # initialize to local attention >>> global_attention_mask = torch.zeros( ... input_ids.shape, dtype=torch.long, device=input_ids.device - >>> ) # initialize to global attention to be deactivated for all tokens + ... ) # initialize to global attention to be deactivated for all tokens >>> global_attention_mask[ ... :, ... [ @@ -1645,7 +1645,7 @@ def forward( ... 4, ... 21, ... ], - >>> ] = 1 # Set global attention to random tokens for the sake of this example + ... ] = 1 # Set global attention to random tokens for the sake of this example >>> # Usually, set global attention based on the task. For example, >>> # classification: the token >>> # QA: question tokens @@ -2025,7 +2025,7 @@ def forward( >>> answer_tokens = all_tokens[torch.argmax(start_logits) : torch.argmax(end_logits) + 1] >>> answer = tokenizer.decode( ... tokenizer.convert_tokens_to_ids(answer_tokens) - >>> ) # remove space prepending space token + ... ) # remove space prepending space token ```""" return_dict = return_dict if return_dict is not None else self.config.use_return_dict diff --git a/src/transformers/models/luke/modeling_luke.py b/src/transformers/models/luke/modeling_luke.py index cd5a53ddae305..7388e2031bab3 100644 --- a/src/transformers/models/luke/modeling_luke.py +++ b/src/transformers/models/luke/modeling_luke.py @@ -953,11 +953,11 @@ def forward( >>> entities = [ ... "Beyoncé", ... "Los Angeles", - >>> ] # Wikipedia entity titles corresponding to the entity mentions "Beyoncé" and "Los Angeles" + ... ] # Wikipedia entity titles corresponding to the entity mentions "Beyoncé" and "Los Angeles" >>> entity_spans = [ ... (0, 7), ... (17, 28), - >>> ] # character-based entity spans corresponding to "Beyoncé" and "Los Angeles" + ... ] # character-based entity spans corresponding to "Beyoncé" and "Los Angeles" >>> encoding = tokenizer( ... text, entities=entities, entity_spans=entity_spans, add_prefix_space=True, return_tensors="pt" @@ -1435,7 +1435,7 @@ def forward( >>> entity_spans = [ ... (0, 7), ... (17, 28), - >>> ] # character-based entity spans corresponding to "Beyoncé" and "Los Angeles" + ... ] # character-based entity spans corresponding to "Beyoncé" and "Los Angeles" >>> inputs = tokenizer(text, entity_spans=entity_spans, return_tensors="pt") >>> outputs = model(**inputs) >>> logits = outputs.logits diff --git a/src/transformers/models/openai/modeling_openai.py b/src/transformers/models/openai/modeling_openai.py index 8ded535cef102..2262db9aa8cf8 100644 --- a/src/transformers/models/openai/modeling_openai.py +++ b/src/transformers/models/openai/modeling_openai.py @@ -674,7 +674,7 @@ def forward( >>> model = OpenAIGPTDoubleHeadsModel.from_pretrained("openai-gpt") >>> tokenizer.add_special_tokens( ... {"cls_token": "[CLS]"} - >>> ) # Add a [CLS] to the vocabulary (we should train it also!) + ... ) # Add a [CLS] to the vocabulary (we should train it also!) >>> model.resize_token_embeddings(len(tokenizer)) >>> choices = ["Hello, my dog is cute [CLS]", "Hello, my cat is cute [CLS]"] diff --git a/src/transformers/models/openai/modeling_tf_openai.py b/src/transformers/models/openai/modeling_tf_openai.py index 5215ad7c2ff6e..24a7935eb0050 100644 --- a/src/transformers/models/openai/modeling_tf_openai.py +++ b/src/transformers/models/openai/modeling_tf_openai.py @@ -693,9 +693,9 @@ def call( >>> inputs = {k: tf.expand_dims(v, 0) for k, v in encoding.items()} >>> inputs["mc_token_ids"] = tf.constant( ... [inputs["input_ids"].shape[-1] - 1, inputs["input_ids"].shape[-1] - 1] - >>> )[ + ... )[ ... None, : - >>> ] # Batch size 1 + ... ] # Batch size 1 >>> outputs = model(inputs) >>> lm_prediction_scores, mc_prediction_scores = outputs[:2] ```""" diff --git a/src/transformers/models/prophetnet/modeling_prophetnet.py b/src/transformers/models/prophetnet/modeling_prophetnet.py index 84fb75f3f3340..c869d6373bc5a 100644 --- a/src/transformers/models/prophetnet/modeling_prophetnet.py +++ b/src/transformers/models/prophetnet/modeling_prophetnet.py @@ -1813,7 +1813,7 @@ def forward( >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="pt" - >>> ).input_ids # Batch size 1 + ... ).input_ids # Batch size 1 >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids # Batch size 1 >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids) @@ -1935,7 +1935,7 @@ def forward( >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="pt" - >>> ).input_ids # Batch size 1 + ... ).input_ids # Batch size 1 >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids # Batch size 1 >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids) @@ -2202,7 +2202,7 @@ def forward( >>> input_ids = tokenizer_enc(ARTICLE, return_tensors="pt").input_ids >>> labels = tokenizer_dec( ... "us rejects charges against its ambassador in bolivia", return_tensors="pt" - >>> ).input_ids + ... ).input_ids >>> outputs = model(input_ids=input_ids, decoder_input_ids=labels[:, :-1], labels=labels[:, 1:]) >>> loss = outputs.loss diff --git a/src/transformers/models/rag/modeling_rag.py b/src/transformers/models/rag/modeling_rag.py index 642b13c580c06..205e825cbc1fa 100644 --- a/src/transformers/models/rag/modeling_rag.py +++ b/src/transformers/models/rag/modeling_rag.py @@ -826,7 +826,7 @@ def forward( >>> docs_dict = retriever(input_ids.numpy(), question_hidden_states.detach().numpy(), return_tensors="pt") >>> doc_scores = torch.bmm( ... question_hidden_states.unsqueeze(1), docs_dict["retrieved_doc_embeds"].float().transpose(1, 2) - >>> ).squeeze(1) + ... ).squeeze(1) >>> # 3. Forward to generator >>> outputs = model( ... context_input_ids=docs_dict["context_input_ids"], @@ -1293,7 +1293,7 @@ def forward( >>> docs_dict = retriever(input_ids.numpy(), question_hidden_states.detach().numpy(), return_tensors="pt") >>> doc_scores = torch.bmm( ... question_hidden_states.unsqueeze(1), docs_dict["retrieved_doc_embeds"].float().transpose(1, 2) - >>> ).squeeze(1) + ... ).squeeze(1) >>> # 3. Forward to generator >>> outputs = model( ... context_input_ids=docs_dict["context_input_ids"], diff --git a/src/transformers/models/rag/retrieval_rag.py b/src/transformers/models/rag/retrieval_rag.py index f39fc48d27c8b..7a3c5635f24f9 100644 --- a/src/transformers/models/rag/retrieval_rag.py +++ b/src/transformers/models/rag/retrieval_rag.py @@ -354,7 +354,7 @@ class RagRetriever: >>> dataset = ( ... ... - >>> ) # dataset must be a datasets.Datasets object with columns "title", "text" and "embeddings", and it must have a faiss index + ... ) # dataset must be a datasets.Datasets object with columns "title", "text" and "embeddings", and it must have a faiss index >>> retriever = RagRetriever.from_pretrained("facebook/dpr-ctx_encoder-single-nq-base", indexed_dataset=dataset) >>> # To load your own indexed dataset built with the datasets library that was saved on disk. More info in examples/rag/use_own_knowledge_dataset.py diff --git a/src/transformers/models/realm/modeling_realm.py b/src/transformers/models/realm/modeling_realm.py index 568eea4f60c52..c467dcd30af83 100644 --- a/src/transformers/models/realm/modeling_realm.py +++ b/src/transformers/models/realm/modeling_realm.py @@ -1787,7 +1787,7 @@ def forward( ... add_special_tokens=False, ... return_token_type_ids=False, ... return_attention_mask=False, - >>> ).input_ids + ... ).input_ids >>> reader_output, predicted_answer_ids = model(**question_ids, answer_ids=answer_ids, return_dict=False) >>> predicted_answer = tokenizer.decode(predicted_answer_ids) diff --git a/src/transformers/models/speech_to_text/modeling_tf_speech_to_text.py b/src/transformers/models/speech_to_text/modeling_tf_speech_to_text.py index 8980636c3b32b..c78d19056bd30 100755 --- a/src/transformers/models/speech_to_text/modeling_tf_speech_to_text.py +++ b/src/transformers/models/speech_to_text/modeling_tf_speech_to_text.py @@ -1387,7 +1387,7 @@ def call( >>> input_features = processor( ... ds["speech"][0], sampling_rate=16000, return_tensors="tf" - >>> ).input_features # Batch size 1 + ... ).input_features # Batch size 1 >>> generated_ids = model.generate(input_features) >>> transcription = processor.batch_decode(generated_ids) diff --git a/src/transformers/models/t5/modeling_flax_t5.py b/src/transformers/models/t5/modeling_flax_t5.py index 263412578c23d..767caea3eb38f 100644 --- a/src/transformers/models/t5/modeling_flax_t5.py +++ b/src/transformers/models/t5/modeling_flax_t5.py @@ -1344,7 +1344,7 @@ class FlaxT5Model(FlaxT5PreTrainedModel): >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="np" - >>> ).input_ids + ... ).input_ids >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="np").input_ids >>> # forward pass diff --git a/src/transformers/models/t5/modeling_t5.py b/src/transformers/models/t5/modeling_t5.py index 9b56e5ba519ac..630e9dd17aa54 100644 --- a/src/transformers/models/t5/modeling_t5.py +++ b/src/transformers/models/t5/modeling_t5.py @@ -1375,7 +1375,7 @@ def forward( >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="pt" - >>> ).input_ids # Batch size 1 + ... ).input_ids # Batch size 1 >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids # Batch size 1 >>> # forward pass @@ -1583,7 +1583,7 @@ def forward( >>> # inference >>> input_ids = tokenizer( ... "summarize: studies have shown that owning a dog is good for you", return_tensors="pt" - >>> ).input_ids # Batch size 1 + ... ).input_ids # Batch size 1 >>> outputs = model.generate(input_ids) >>> print(tokenizer.decode(outputs[0], skip_special_tokens=True)) >>> # studies have shown that owning a dog is good for you. @@ -1831,7 +1831,7 @@ def forward( >>> model = T5EncoderModel.from_pretrained("t5-small") >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="pt" - >>> ).input_ids # Batch size 1 + ... ).input_ids # Batch size 1 >>> outputs = model(input_ids=input_ids) >>> last_hidden_states = outputs.last_hidden_state ```""" diff --git a/src/transformers/models/t5/modeling_tf_t5.py b/src/transformers/models/t5/modeling_tf_t5.py index 2e48174a9048b..3434a6ea4f377 100644 --- a/src/transformers/models/t5/modeling_tf_t5.py +++ b/src/transformers/models/t5/modeling_tf_t5.py @@ -1165,7 +1165,7 @@ def call( >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="tf" - >>> ).input_ids # Batch size 1 + ... ).input_ids # Batch size 1 >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="tf").input_ids # Batch size 1 >>> # forward pass @@ -1353,7 +1353,7 @@ def call( >>> # inference >>> inputs = tokenizer( ... "summarize: studies have shown that owning a dog is good for you", return_tensors="tf" - >>> ).input_ids # Batch size 1 + ... ).input_ids # Batch size 1 >>> outputs = model.generate(inputs) >>> print(tokenizer.decode(outputs[0], skip_special_tokens=True)) >>> # studies have shown that owning a dog is good for you @@ -1642,7 +1642,7 @@ def call( >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="tf" - >>> ).input_ids # Batch size 1 + ... ).input_ids # Batch size 1 >>> outputs = model(input_ids) ```""" diff --git a/src/transformers/models/tapas/modeling_tapas.py b/src/transformers/models/tapas/modeling_tapas.py index 12703cc2dfaaa..b0c3786ca05a4 100644 --- a/src/transformers/models/tapas/modeling_tapas.py +++ b/src/transformers/models/tapas/modeling_tapas.py @@ -1068,7 +1068,7 @@ def forward( ... ) >>> labels = tokenizer( ... table=table, queries="How many movies has George Clooney played in?", return_tensors="pt" - >>> )["input_ids"] + ... )["input_ids"] >>> outputs = model(**inputs, labels=labels) >>> logits = outputs.logits diff --git a/src/transformers/models/tapas/modeling_tf_tapas.py b/src/transformers/models/tapas/modeling_tf_tapas.py index e91baaab8edb2..d2da0644627ac 100644 --- a/src/transformers/models/tapas/modeling_tf_tapas.py +++ b/src/transformers/models/tapas/modeling_tf_tapas.py @@ -1095,7 +1095,7 @@ def call( ... ) >>> labels = tokenizer( ... table=table, queries="How many movies has George Clooney played in?", return_tensors="tf" - >>> )["input_ids"] + ... )["input_ids"] >>> outputs = model(**inputs, labels=labels) >>> logits = outputs.logits diff --git a/src/transformers/models/vision_encoder_decoder/modeling_tf_vision_encoder_decoder.py b/src/transformers/models/vision_encoder_decoder/modeling_tf_vision_encoder_decoder.py index edc2973a07348..6bbf514091038 100644 --- a/src/transformers/models/vision_encoder_decoder/modeling_tf_vision_encoder_decoder.py +++ b/src/transformers/models/vision_encoder_decoder/modeling_tf_vision_encoder_decoder.py @@ -326,7 +326,7 @@ def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): >>> output_ids = model.generate( ... pixel_values, max_length=16, num_beams=4, return_dict_in_generate=True - >>> ).sequences + ... ).sequences >>> preds = decoder_tokenizer.batch_decode(output_ids, skip_special_tokens=True) >>> preds = [pred.strip() for pred in preds] diff --git a/src/transformers/models/wav2vec2/modeling_flax_wav2vec2.py b/src/transformers/models/wav2vec2/modeling_flax_wav2vec2.py index 1386ca37b075d..7709e43ab9556 100644 --- a/src/transformers/models/wav2vec2/modeling_flax_wav2vec2.py +++ b/src/transformers/models/wav2vec2/modeling_flax_wav2vec2.py @@ -1081,7 +1081,7 @@ class FlaxWav2Vec2Model(FlaxWav2Vec2PreTrainedModel): >>> input_values = processor( ... ds["speech"][0], sampling_rate=16_000, return_tensors="np" - >>> ).input_values # Batch size 1 + ... ).input_values # Batch size 1 >>> hidden_states = model(input_values).last_hidden_state ``` """ @@ -1200,7 +1200,7 @@ class FlaxWav2Vec2ForCTC(FlaxWav2Vec2PreTrainedModel): >>> input_values = processor( ... ds["speech"][0], sampling_rate=16_000, return_tensors="np" - >>> ).input_values # Batch size 1 + ... ).input_values # Batch size 1 >>> logits = model(input_values).logits >>> predicted_ids = jnp.argmax(logits, axis=-1) diff --git a/src/transformers/models/xlm/modeling_xlm.py b/src/transformers/models/xlm/modeling_xlm.py index ed0817afbb895..ebb3c503475c0 100755 --- a/src/transformers/models/xlm/modeling_xlm.py +++ b/src/transformers/models/xlm/modeling_xlm.py @@ -1039,7 +1039,7 @@ def forward( >>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze( ... 0 - >>> ) # Batch size 1 + ... ) # Batch size 1 >>> start_positions = torch.tensor([1]) >>> end_positions = torch.tensor([3]) diff --git a/src/transformers/models/xlm_prophetnet/modeling_xlm_prophetnet.py b/src/transformers/models/xlm_prophetnet/modeling_xlm_prophetnet.py index dfb7b394915b5..8961fbbfc3740 100644 --- a/src/transformers/models/xlm_prophetnet/modeling_xlm_prophetnet.py +++ b/src/transformers/models/xlm_prophetnet/modeling_xlm_prophetnet.py @@ -98,7 +98,7 @@ class XLMProphetNetModel(ProphetNetModel): >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="pt" - >>> ).input_ids # Batch size 1 + ... ).input_ids # Batch size 1 >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids # Batch size 1 >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids) @@ -124,7 +124,7 @@ class XLMProphetNetForConditionalGeneration(ProphetNetForConditionalGeneration): >>> input_ids = tokenizer( ... "Studies have been shown that owning a dog is good for you", return_tensors="pt" - >>> ).input_ids # Batch size 1 + ... ).input_ids # Batch size 1 >>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids # Batch size 1 >>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids) diff --git a/src/transformers/models/xlnet/modeling_tf_xlnet.py b/src/transformers/models/xlnet/modeling_tf_xlnet.py index f5a1cba3c837f..df4111d263172 100644 --- a/src/transformers/models/xlnet/modeling_tf_xlnet.py +++ b/src/transformers/models/xlnet/modeling_tf_xlnet.py @@ -1281,17 +1281,17 @@ def call( >>> # We show how to setup inputs to predict a next token using a bi-directional context. >>> input_ids = tf.constant(tokenizer.encode("Hello, my dog is very ", add_special_tokens=True))[ ... None, : - >>> ] # We will predict the masked token + ... ] # We will predict the masked token >>> perm_mask = np.zeros((1, input_ids.shape[1], input_ids.shape[1])) >>> perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token >>> target_mapping = np.zeros( ... (1, 1, input_ids.shape[1]) - >>> ) # Shape [1, 1, seq_length] => let's predict one token + ... ) # Shape [1, 1, seq_length] => let's predict one token >>> target_mapping[ ... 0, 0, -1 - >>> ] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) + ... ] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) >>> outputs = model( ... input_ids, @@ -1301,7 +1301,7 @@ def call( >>> next_token_logits = outputs[ ... 0 - >>> ] # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] + ... ] # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] ```""" transformer_outputs = self.transformer( input_ids=input_ids, diff --git a/src/transformers/models/xlnet/modeling_xlnet.py b/src/transformers/models/xlnet/modeling_xlnet.py index 079c636628f2f..dc7f78eeb8e2f 100755 --- a/src/transformers/models/xlnet/modeling_xlnet.py +++ b/src/transformers/models/xlnet/modeling_xlnet.py @@ -1400,47 +1400,47 @@ def forward( >>> # We show how to setup inputs to predict a next token using a bi-directional context. >>> input_ids = torch.tensor( ... tokenizer.encode("Hello, my dog is very ", add_special_tokens=False) - >>> ).unsqueeze( + ... ).unsqueeze( ... 0 - >>> ) # We will predict the masked token + ... ) # We will predict the masked token >>> perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float) >>> perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token >>> target_mapping = torch.zeros( ... (1, 1, input_ids.shape[1]), dtype=torch.float - >>> ) # Shape [1, 1, seq_length] => let's predict one token + ... ) # Shape [1, 1, seq_length] => let's predict one token >>> target_mapping[ ... 0, 0, -1 - >>> ] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) + ... ] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) >>> outputs = model(input_ids, perm_mask=perm_mask, target_mapping=target_mapping) >>> next_token_logits = outputs[ ... 0 - >>> ] # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] + ... ] # Output has shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] >>> # The same way can the XLNetLMHeadModel be used to be trained by standard auto-regressive language modeling. >>> input_ids = torch.tensor( ... tokenizer.encode("Hello, my dog is very ", add_special_tokens=False) - >>> ).unsqueeze( + ... ).unsqueeze( ... 0 - >>> ) # We will predict the masked token + ... ) # We will predict the masked token >>> labels = torch.tensor(tokenizer.encode("cute", add_special_tokens=False)).unsqueeze(0) >>> assert labels.shape[0] == 1, "only one word will be predicted" >>> perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float) >>> perm_mask[ ... :, :, -1 - >>> ] = 1.0 # Previous tokens don't see last token as is done in standard auto-regressive lm training + ... ] = 1.0 # Previous tokens don't see last token as is done in standard auto-regressive lm training >>> target_mapping = torch.zeros( ... (1, 1, input_ids.shape[1]), dtype=torch.float - >>> ) # Shape [1, 1, seq_length] => let's predict one token + ... ) # Shape [1, 1, seq_length] => let's predict one token >>> target_mapping[ ... 0, 0, -1 - >>> ] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) + ... ] = 1.0 # Our first (and only) prediction will be the last token of the sequence (the masked token) >>> outputs = model(input_ids, perm_mask=perm_mask, target_mapping=target_mapping, labels=labels) >>> loss = outputs.loss >>> next_token_logits = ( ... outputs.logits - >>> ) # Logits have shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] + ... ) # Logits have shape [target_mapping.size(0), target_mapping.size(1), config.vocab_size] ```""" return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1980,7 +1980,7 @@ def forward( >>> input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze( ... 0 - >>> ) # Batch size 1 + ... ) # Batch size 1 >>> start_positions = torch.tensor([1]) >>> end_positions = torch.tensor([3]) >>> outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) From f1fe820ddb8652de856a01bde238a80c7b7d8174 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Fri, 6 May 2022 18:39:09 +0200 Subject: [PATCH 041/145] fixed embeddings and generation tests --- tests/models/opt/test_modeling_opt.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index 4419ecdbf26fd..520eb72f856af 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -824,7 +824,6 @@ def setUp(self): self.path_model = "ArthurZ/opt-350m" # TODO add the model on the hub # self.path_logits_meta = "/home/younes/Desktop/Work/metaseq-conversion/logits_metaseq_gpt2_tokenizer.p" # TODO add the logits somewhere? - # @unittest.skip("Skipping unless we find a solution to store the model") def test_load_model(self): try: _ = OPTForCausalLM.from_pretrained(self.path_model) @@ -866,7 +865,6 @@ def setUp(self): self.path_model = "ArthurZ/opt-350m" # self.path_logits_meta = "/home/younes/Desktop/Work/metaseq-conversion/logits_metaseq_gpt2_tokenizer.p" # TODO add the logits somewhere? - @unittest.skip("Skipping unless we find a solution to store the model") def test_generation(self): model = OPTForCausalLM.from_pretrained(self.path_model) model = model.eval() From 9b9c65b777a2904e8389cc56e512e115058f8524 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Sat, 7 May 2022 07:34:52 +0200 Subject: [PATCH 042/145] update paths --- tests/models/opt/test_modeling_opt.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index 520eb72f856af..240112cc2142c 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -508,11 +508,11 @@ def test_encoder_equiv(self): class OPTModelIntegrationTests(unittest.TestCase): @cached_property def default_tokenizer(self): - return GPT2Tokenizer.from_pretrained("facebook/opt-large") + return GPT2Tokenizer.from_pretrained("patrickvonplaten/opt_gpt2_tokenizer") @slow def test_inference_no_head(self): - model = OPTModel.from_pretrained("facebook/opt-large").to(torch_device) + model = OPTModel.from_pretrained("ArthurZ/350-m").to(torch_device) input_ids = _long_tensor([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]) attention_mask = input_ids.ne(model.config.pad_token_id) with torch.no_grad(): @@ -567,11 +567,11 @@ def test_inference_no_head(self): # assert_tensors_close(batched_logits[1], logits2, atol=1e-3) # assert_tensors_close(expected_slice, logits_arr, atol=1e-3) - def test_xsum_config_generation_params(self): - config = OPTConfig.from_pretrained("facebook/opt-large-xsum") - expected_params = dict(num_beams=6, do_sample=False, early_stopping=True, length_penalty=1.0) - config_params = {k: getattr(config, k, "MISSING") for k, v in expected_params.items()} - self.assertDictEqual(expected_params, config_params) + # def test_xsum_config_generation_params(self): + # config = OPTConfig.from_pretrained("facebook/opt-large-xsum") + # expected_params = dict(num_beams=6, do_sample=False, early_stopping=True, length_penalty=1.0) + # config_params = {k: getattr(config, k, "MISSING") for k, v in expected_params.items()} + # self.assertDictEqual(expected_params, config_params) class OPTStandaloneDecoderModelTester: From 4fb9608a5f31856152746981c7e6aa3d76ae3043 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Sat, 7 May 2022 08:02:31 +0200 Subject: [PATCH 043/145] clean comments --- tests/models/opt/test_modeling_opt.py | 34 --------------------------- 1 file changed, 34 deletions(-) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index 240112cc2142c..be7f88c35d704 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -177,40 +177,6 @@ def create_and_check_decoder_model_past_large_inputs(self, config, inputs_dict): # test that outputs are equal for slice self.parent.assertTrue(torch.allclose(output_from_past_slice, output_from_no_past_slice, atol=1e-3)) - # Do not use this test since it is for encoder decoders? - - # def check_encoder_decoder_model_standalone(self, config, inputs_dict): - # model = OPTModel(config=config).to(torch_device).eval() - # outputs = model(**inputs_dict) - - # encoder_last_hidden_state = outputs.encoder_last_hidden_state - # last_hidden_state = outputs.last_hidden_state - - # with tempfile.TemporaryDirectory() as tmpdirname: - # encoder = model.get_encoder() - # encoder.save_pretrained(tmpdirname) - # encoder = OPTEncoder.from_pretrained(tmpdirname).to(torch_device) - - # encoder_last_hidden_state_2 = encoder(inputs_dict["input_ids"], attention_mask=inputs_dict["attention_mask"])[ - # 0 - # ] - - # self.parent.assertTrue((encoder_last_hidden_state_2 - encoder_last_hidden_state).abs().max().item() < 1e-3) - - # with tempfile.TemporaryDirectory() as tmpdirname: - # decoder = model.get_decoder() - # decoder.save_pretrained(tmpdirname) - # decoder = OPTDecoder.from_pretrained(tmpdirname).to(torch_device) - - # last_hidden_state_2 = decoder( - # input_ids=inputs_dict["decoder_input_ids"], - # attention_mask=inputs_dict["decoder_attention_mask"], - # encoder_hidden_states=encoder_last_hidden_state, - # encoder_attention_mask=inputs_dict["attention_mask"], - # )[0] - - # self.parent.assertTrue((last_hidden_state_2 - last_hidden_state).abs().max().item() < 1e-3) - @require_torch class OPTHeadTests(unittest.TestCase): From ab57047fa42bca9c0f8030d7905964f870eb133f Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Sun, 8 May 2022 14:03:37 +0200 Subject: [PATCH 044/145] removed OPTClassificationHead for sentence classification --- src/transformers/models/opt/modeling_opt.py | 24 +-------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 0a736092bcf97..29ac89eec05fd 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -52,7 +52,7 @@ OPT_PRETRAINED_MODEL_ARCHIVE_LIST = [ - "", + "opt350-m", # See all OPT models at https://huggingface.co/models?filter=opt ] @@ -445,28 +445,6 @@ def forward( return outputs -class OPTClassificationHead(nn.Module): - """Head for sentence-level classification tasks.""" - - def __init__( - self, - input_dim: int, - inner_dim: int, - num_classes: int, - pooler_dropout: float, - ): - super().__init__() - self.dense = nn.Linear(input_dim, inner_dim) - self.dropout = nn.Dropout(p=pooler_dropout) - self.out_proj = nn.Linear(inner_dim, num_classes) - - def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: - hidden_states = self.dropout(hidden_states) - hidden_states = self.dense(hidden_states) - hidden_states = torch.tanh(hidden_states) - hidden_states = self.dropout(hidden_states) - hidden_states = self.out_proj(hidden_states) - return hidden_states OPT_START_DOCSTRING = r""" From 0c1c79130b64457d83d37cceb76641fba691be78 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Mon, 9 May 2022 09:33:03 +0200 Subject: [PATCH 045/145] renamed hidden layer --- .../models/opt/configuration_opt.py | 26 +++++++++---------- tests/models/opt/test_modeling_opt.py | 10 +++---- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/transformers/models/opt/configuration_opt.py b/src/transformers/models/opt/configuration_opt.py index a10fbcd18e3fc..dca6aef537d5b 100644 --- a/src/transformers/models/opt/configuration_opt.py +++ b/src/transformers/models/opt/configuration_opt.py @@ -48,7 +48,7 @@ class OPTConfig(PretrainedConfig): `inputs_ids` passed when calling [`OPTModel`] or [`TFOPTModel`]. d_model (`int`, *optional*, defaults to 1024): Dimensionality of the layers and the pooler layer. - num_layers (`int`, *optional*, defaults to 12): + num_hidden_layers (`int`, *optional*, defaults to 12): Number of decoder layers. ffn_dim (`int`, *optional*, defaults to 4096): Dimensionality of the "intermediate" (often named feed-forward) layer in decoder. @@ -95,13 +95,13 @@ class OPTConfig(PretrainedConfig): ```""" model_type = "opt" keys_to_ignore_at_inference = ["past_key_values"] - attribute_map = {"hidden_size": "d_model", "num_hidden_layers": "num_layers"} + attribute_map = {"hidden_size": "d_model", "num_hidden_layers": "num_hidden_layers"} def __init__( self, vocab_size=50272, # TODO check the real value max_position_embeddings=2048, - num_layers=24, + num_hidden_layers=24, num_attention_heads=16, ffn_dim=4096, layerdrop=0.0, @@ -129,7 +129,7 @@ def __init__( self.ffn_dim = ffn_dim self.share_input_output_embed = share_input_output_embed self.d_model = d_model - self.num_layers = num_layers + self.num_hidden_layers = num_hidden_layers self.dropout = dropout self.attention_dropout = attention_dropout self.activation_dropout = activation_dropout @@ -186,7 +186,7 @@ def inputs(self) -> Mapping[str, Mapping[int, str]]: ] ) if self.use_past: - num_encoder_layers, _ = self.num_layers + num_encoder_layers, _ = self.num_hidden_layers for i in range(num_encoder_layers): common_inputs[f"past_key_values.{i}.key"] = {0: "batch", 2: "past_sequence + sequence"} common_inputs[f"past_key_values.{i}.value"] = {0: "batch", 2: "past_sequence + sequence"} @@ -209,7 +209,7 @@ def outputs(self) -> Mapping[str, Mapping[int, str]]: else: common_outputs = super(OnnxConfigWithPast, self).outputs if self.use_past: - num_encoder_layers, _ = self.num_layers + num_encoder_layers, _ = self.num_hidden_layers for i in range(num_encoder_layers): common_outputs[f"present.{i}.key"] = {0: "batch", 2: "past_sequence + sequence"} common_outputs[f"present.{i}.value"] = {0: "batch", 2: "past_sequence + sequence"} @@ -263,12 +263,12 @@ def _generate_dummy_inputs_for_default_and_seq2seq_lm( common_inputs["past_key_values"] = [] # If the number of encoder and decoder layers are present in the model configuration, both are considered - num_encoder_layers, num_layers = self.num_layers - min_num_layers = min(num_encoder_layers, num_layers) - max_num_layers = max(num_encoder_layers, num_layers) - min_num_layers - remaining_side_name = "encoder" if num_encoder_layers > num_layers else "decoder" + num_encoder_layers, num_hidden_layers = self.num_hidden_layers + min_num_hidden_layers = min(num_encoder_layers, num_hidden_layers) + max_num_hidden_layers = max(num_encoder_layers, num_hidden_layers) - min_num_hidden_layers + remaining_side_name = "encoder" if num_encoder_layers > num_hidden_layers else "decoder" - for _ in range(min_num_layers): + for _ in range(min_num_hidden_layers): common_inputs["past_key_values"].append( ( torch.zeros(decoder_shape), @@ -279,7 +279,7 @@ def _generate_dummy_inputs_for_default_and_seq2seq_lm( ) # TODO: test this. shape = encoder_shape if remaining_side_name == "encoder" else decoder_shape - for _ in range(min_num_layers, max_num_layers): + for _ in range(min_num_hidden_layers, max_num_hidden_layers): common_inputs["past_key_values"].append((torch.zeros(shape), torch.zeros(shape))) return common_inputs @@ -303,7 +303,7 @@ def _generate_dummy_inputs_for_causal_lm( batch, seqlen = common_inputs["input_ids"].shape # Not using the same length for past_key_values past_key_values_length = seqlen + 2 - num_encoder_layers, _ = self.num_layers + num_encoder_layers, _ = self.num_hidden_layers num_encoder_attention_heads, _ = self.num_attention_heads past_shape = ( batch, diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index be7f88c35d704..7763ab2b27e1d 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -52,9 +52,9 @@ def prepare_opt_inputs_dict( if decoder_attention_mask is None: decoder_attention_mask = decoder_input_ids.ne(config.pad_token_id) if decoder_head_mask is None: - decoder_head_mask = torch.ones(config.decoder_layers, config.decoder_attention_heads, device=torch_device) + decoder_head_mask = torch.ones(config.num_hidden_layers, config.num_attention_head, device=torch_device) if cross_attn_head_mask is None: - cross_attn_head_mask = torch.ones(config.decoder_layers, config.decoder_attention_heads, device=torch_device) + cross_attn_head_mask = torch.ones(config.num_hidden_layers, config.num_attention_head, device=torch_device) return { "input_ids": input_ids, "decoder_input_ids": decoder_input_ids, @@ -123,7 +123,7 @@ def get_config(self): return OPTConfig( vocab_size=self.vocab_size, d_model=self.hidden_size, - num_layers=self.num_hidden_layers, + num_hidden_layers=self.num_hidden_layers, num_attention_heads=self.num_attention_heads, ffn_dim=self.intermediate_size, dropout=self.hidden_dropout_prob, @@ -207,7 +207,7 @@ def _get_config_and_data(self): config = OPTConfig( vocab_size=self.vocab_size, d_model=24, - num_layers=2, + num_hidden_layers=2, num_attention_heads=2, ffn_dim=32, max_position_embeddings=48, @@ -233,7 +233,7 @@ def test_generate_beam_search(self): config = OPTConfig( vocab_size=self.vocab_size, d_model=24, - num_layers=2, + num_hidden_layers=2, num_attention_heads=2, ffn_dim=32, max_position_embeddings=48, From ac50b44809fc21512e1677f0503a0403941cd6e6 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Mon, 9 May 2022 09:33:43 +0200 Subject: [PATCH 046/145] renamed num layers to standard num_hidden_layers --- src/transformers/models/opt/modeling_opt.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 29ac89eec05fd..0b31c95a96a95 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -583,7 +583,7 @@ def dummy_inputs(self): class OPTDecoder(OPTPretrainedModel): """ - Transformer decoder consisting of *config.num_layers* layers. Each layer is a [`OPTDecoderLayer`] + Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`OPTDecoderLayer`] Args: config: OPTConfig @@ -621,7 +621,7 @@ def __init__(self, config: OPTConfig): self.output_projection = nn.Linear(config.embed_dim, config.vocab_size, bias=False) - self.layers = nn.ModuleList([OPTDecoderLayer(config) for _ in range(config.num_layers)]) + self.layers = nn.ModuleList([OPTDecoderLayer(config) for _ in range(config.num_hidden_layers)]) self.gradient_checkpointing = False # Initialize weights and apply final processing @@ -682,13 +682,13 @@ def forward( - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) - head_mask (`torch.Tensor` of shape `(num_layers, num_attention_heads)`, *optional*): + head_mask (`torch.Tensor` of shape `(num_hidden_layers, num_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. - cross_attn_head_mask (`torch.Tensor` of shape `(num_layers, num_attention_heads)`, *optional*): + cross_attn_head_mask (`torch.Tensor` of shape `(num_hidden_layers, num_attention_heads)`, *optional*): Mask to nullify selected heads of the cross-attention modules in the decoder to avoid performing cross-attention on hidden heads. Mask values selected in `[0, 1]`: @@ -1022,13 +1022,13 @@ def forward( - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) - head_mask (`torch.Tensor` of shape `(num_layers, num_attention_heads)`, *optional*): + head_mask (`torch.Tensor` of shape `(num_hidden_layers, num_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. - cross_attn_head_mask (`torch.Tensor` of shape `(num_layers, num_attention_heads)`, *optional*): + cross_attn_head_mask (`torch.Tensor` of shape `(num_hidden_layers, num_attention_heads)`, *optional*): Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, From 1505de51a66b89ecf241dbe0329babc21ef0bb43 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Mon, 9 May 2022 10:00:47 +0200 Subject: [PATCH 047/145] num_attention_heads fix --- tests/models/opt/test_modeling_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index 7763ab2b27e1d..208efedd5f859 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -52,9 +52,9 @@ def prepare_opt_inputs_dict( if decoder_attention_mask is None: decoder_attention_mask = decoder_input_ids.ne(config.pad_token_id) if decoder_head_mask is None: - decoder_head_mask = torch.ones(config.num_hidden_layers, config.num_attention_head, device=torch_device) + decoder_head_mask = torch.ones(config.num_hidden_layers, config.num_attention_heads, device=torch_device) if cross_attn_head_mask is None: - cross_attn_head_mask = torch.ones(config.num_hidden_layers, config.num_attention_head, device=torch_device) + cross_attn_head_mask = torch.ones(config.num_hidden_layers, config.num_attention_heads, device=torch_device) return { "input_ids": input_ids, "decoder_input_ids": decoder_input_ids, From 8ace67b4cf7dbdf25590c9d5fff48c4410832257 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Mon, 9 May 2022 10:03:55 +0200 Subject: [PATCH 048/145] changes for 125m --- .../models/opt/configuration_opt.py | 6 +++++ ..._original_pytorch_checkpoint_to_pytorch.py | 4 +++- src/transformers/models/opt/modeling_opt.py | 23 +++++++++++++------ 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/src/transformers/models/opt/configuration_opt.py b/src/transformers/models/opt/configuration_opt.py index a10fbcd18e3fc..7f29daeacff23 100644 --- a/src/transformers/models/opt/configuration_opt.py +++ b/src/transformers/models/opt/configuration_opt.py @@ -120,6 +120,9 @@ def __init__( eos_token_id=2, decoder_start_token_id=2, forced_eos_token_id=2, + project_in=True, + project_out=True, + output_projection=True, **kwargs ): self.vocab_size = vocab_size @@ -138,6 +141,9 @@ def __init__( self.layerdrop = layerdrop self.use_cache = use_cache self.scale_embedding = scale_embedding # scale factor will be sqrt(d_model) if True + self.project_in = project_in + self.project_out = project_out + self.output_projection = output_projection super().__init__( pad_token_id=pad_token_id, diff --git a/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py index ac29b24a5b7cc..82dc147dbf29a 100644 --- a/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py @@ -63,7 +63,9 @@ def rename_key(dct, old, new): def load_xsum_checkpoint(checkpoint_path): """Checkpoint path should end in model.pt""" - sd = torch.load(checkpoint_path, map_location="cpu")["model"] + sd = torch.load(checkpoint_path, map_location="cpu") + if "model" in sd.keys(): + sd = torch.load(checkpoint_path, map_location="cpu")["model"] sd.pop("decoder.version") return sd diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 0a736092bcf97..63b70ee73f909 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -633,15 +633,24 @@ def __init__(self, config: OPTConfig): self.padding_idx, ) - self.project_out_dim = ( - nn.Linear(config.d_model, config.embed_dim, bias=False) if config.embed_dim != config.d_model else None - ) + if config.project_out: + self.project_out_dim = ( + nn.Linear(config.d_model, config.embed_dim, bias=False) if config.embed_dim != config.d_model else None + ) + else: + self.project_out_dim = None - self.project_in_dim = ( - nn.Linear(config.embed_dim, config.d_model, bias=False) if config.d_model != config.embed_dim else None - ) + if config.project_in: + self.project_in_dim = ( + nn.Linear(config.embed_dim, config.d_model, bias=False) if config.d_model != config.embed_dim else None + ) + else: + self.project_in_dim = None - self.output_projection = nn.Linear(config.embed_dim, config.vocab_size, bias=False) + if config.output_projection: + self.output_projection = nn.Linear(config.embed_dim, config.vocab_size, bias=False) + else: + self.output_projection = None self.layers = nn.ModuleList([OPTDecoderLayer(config) for _ in range(config.num_layers)]) From 752c1d283a9edb2887af6ea13ac80ddf7617551b Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Mon, 9 May 2022 10:19:55 +0200 Subject: [PATCH 049/145] add first version for 125m --- .../models/opt/configuration_opt.py | 18 +++++++-------- src/transformers/models/opt/modeling_opt.py | 23 +++++++++---------- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/src/transformers/models/opt/configuration_opt.py b/src/transformers/models/opt/configuration_opt.py index 69b234f052e2e..e5648570ce847 100644 --- a/src/transformers/models/opt/configuration_opt.py +++ b/src/transformers/models/opt/configuration_opt.py @@ -101,13 +101,13 @@ def __init__( self, vocab_size=50272, # TODO check the real value max_position_embeddings=2048, - num_hidden_layers=24, - num_attention_heads=16, - ffn_dim=4096, + num_hidden_layers=12, + num_attention_heads=12, + ffn_dim=3072, layerdrop=0.0, activation_function="gelu", - d_model=1024, - embed_dim=512, + d_model=768, + embed_dim=768, dropout=0.1, attention_dropout=0.0, activation_dropout=0.0, @@ -120,9 +120,8 @@ def __init__( eos_token_id=2, decoder_start_token_id=2, forced_eos_token_id=2, - project_in=True, - project_out=True, - output_projection=True, + output_projection=False, + decoder_layernorm=True, **kwargs ): self.vocab_size = vocab_size @@ -141,9 +140,8 @@ def __init__( self.layerdrop = layerdrop self.use_cache = use_cache self.scale_embedding = scale_embedding # scale factor will be sqrt(d_model) if True - self.project_in = project_in - self.project_out = project_out self.output_projection = output_projection + self.decoder_layernorm = decoder_layernorm super().__init__( pad_token_id=pad_token_id, diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 6170832a73f41..41e7e695bb779 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -611,19 +611,15 @@ def __init__(self, config: OPTConfig): self.padding_idx, ) - if config.project_out: - self.project_out_dim = ( - nn.Linear(config.d_model, config.embed_dim, bias=False) if config.embed_dim != config.d_model else None - ) - else: - self.project_out_dim = None + self.project_out_dim = ( + nn.Linear(config.d_model, config.embed_dim, bias=False) if config.embed_dim != config.d_model else None + ) - if config.project_in: - self.project_in_dim = ( - nn.Linear(config.embed_dim, config.d_model, bias=False) if config.d_model != config.embed_dim else None - ) - else: - self.project_in_dim = None + self.project_in_dim = ( + nn.Linear(config.embed_dim, config.d_model, bias=False) if config.d_model != config.embed_dim else None + ) + + self.layer_norm = nn.LayerNorm(config.d_model) if config.decoder_layernorm else None if config.output_projection: self.output_projection = nn.Linear(config.embed_dim, config.vocab_size, bias=False) @@ -829,6 +825,9 @@ def custom_forward(*inputs): use_cache=use_cache, key_padding_mask=self_attn_padding_mask, ) + + if self.layer_norm: + layer_outputs = self.layer_norm(layer_outputs) hidden_states = layer_outputs[0] if use_cache: From 77e6e04a766be1c0ffe4694dbf88ca86bf9c8890 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Mon, 9 May 2022 10:27:05 +0200 Subject: [PATCH 050/145] add first version - flax --- .../models/opt/modeling_flax_opt.py | 252 +++--------------- 1 file changed, 35 insertions(+), 217 deletions(-) diff --git a/src/transformers/models/opt/modeling_flax_opt.py b/src/transformers/models/opt/modeling_flax_opt.py index 2bc8412b5b257..42ae856f7f104 100644 --- a/src/transformers/models/opt/modeling_flax_opt.py +++ b/src/transformers/models/opt/modeling_flax_opt.py @@ -240,7 +240,7 @@ def setup(self) -> None: f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}" f" and `num_heads`: {self.num_heads})." ) - + self.scaling = self.head_dim**-0.5 dense = partial( nn.Dense, self.embed_dim, @@ -313,7 +313,7 @@ def __call__( batch_size = hidden_states.shape[0] # get query proj - query_states = self.q_proj(hidden_states) + query_states = self.q_proj(hidden_states) * self.scaling # get key, value proj if is_cross_attention: # cross_attentions @@ -391,118 +391,6 @@ def __call__( return attn_output, attn_weights -# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartEncoderLayer with Bart->OPT -class FlaxOPTEncoderLayer(nn.Module): - config: OPTConfig - dtype: jnp.dtype = jnp.float32 - - def setup(self) -> None: - self.embed_dim = self.config.d_model - self.self_attn = FlaxOPTAttention( - config=self.config, - embed_dim=self.embed_dim, - num_heads=self.config.encoder_attention_heads, - dropout=self.config.attention_dropout, - dtype=self.dtype, - ) - self.self_attn_layer_norm = nn.LayerNorm(dtype=self.dtype, epsilon=1e-05) - self.dropout_layer = nn.Dropout(rate=self.config.dropout) - self.activation_fn = ACT2FN[self.config.activation_function] - self.activation_dropout_layer = nn.Dropout(rate=self.config.activation_dropout) - self.fc1 = nn.Dense( - self.config.encoder_ffn_dim, - dtype=self.dtype, - kernel_init=jax.nn.initializers.normal(self.config.init_std), - ) - self.fc2 = nn.Dense( - self.embed_dim, dtype=self.dtype, kernel_init=jax.nn.initializers.normal(self.config.init_std) - ) - self.final_layer_norm = nn.LayerNorm(dtype=self.dtype, epsilon=1e-05) - - def __call__( - self, - hidden_states: jnp.ndarray, - attention_mask: jnp.ndarray, - output_attentions: bool = True, - deterministic: bool = True, - ) -> Tuple[jnp.ndarray]: - residual = hidden_states - hidden_states, attn_weights = self.self_attn(hidden_states=hidden_states, attention_mask=attention_mask) - - hidden_states = self.dropout_layer(hidden_states, deterministic=deterministic) - hidden_states = residual + hidden_states - hidden_states = self.self_attn_layer_norm(hidden_states) - - residual = hidden_states - hidden_states = self.activation_fn(self.fc1(hidden_states)) - hidden_states = self.activation_dropout_layer(hidden_states, deterministic=deterministic) - hidden_states = self.fc2(hidden_states) - hidden_states = self.dropout_layer(hidden_states, deterministic=deterministic) - hidden_states = residual + hidden_states - hidden_states = self.final_layer_norm(hidden_states) - - outputs = (hidden_states,) - - if output_attentions: - outputs += (attn_weights,) - - return outputs - - -# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartEncoderLayerCollection with Bart->OPT -class FlaxOPTEncoderLayerCollection(nn.Module): - config: OPTConfig - dtype: jnp.dtype = jnp.float32 # the dtype of the computation - - def setup(self): - self.layers = [ - FlaxOPTEncoderLayer(self.config, name=str(i), dtype=self.dtype) for i in range(self.config.encoder_layers) - ] - self.layerdrop = self.config.encoder_layerdrop - - def __call__( - self, - hidden_states, - attention_mask, - deterministic: bool = True, - output_attentions: bool = False, - output_hidden_states: bool = False, - return_dict: bool = True, - ): - all_attentions = () if output_attentions else None - all_hidden_states = () if output_hidden_states else None - - for encoder_layer in self.layers: - if output_hidden_states: - all_hidden_states = all_hidden_states + (hidden_states,) - # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description) - dropout_probability = random.uniform(0, 1) - if not deterministic and (dropout_probability < self.layerdrop): # skip the layer - layer_outputs = (None, None) - else: - layer_outputs = encoder_layer( - hidden_states, - attention_mask, - output_attentions, - deterministic, - ) - hidden_states = layer_outputs[0] - if output_attentions: - all_attentions = all_attentions + (layer_outputs[1],) - - if output_hidden_states: - all_hidden_states += (hidden_states,) - - outputs = (hidden_states, all_hidden_states, all_attentions) - - if not return_dict: - return tuple(v for v in outputs if v is not None) - - return FlaxBaseModelOutput( - last_hidden_state=hidden_states, hidden_states=all_hidden_states, attentions=all_attentions - ) - - # Copied from transformers.models.bart.modeling_flax_bart.FlaxBartDecoderLayer with Bart->OPT class FlaxOPTDecoderLayer(nn.Module): config: OPTConfig @@ -692,71 +580,6 @@ def __call__(self, hidden_states: jnp.ndarray, deterministic: bool): return hidden_states -# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartEncoder with Bart->OPT -class FlaxOPTEncoder(nn.Module): - config: OPTConfig - embed_tokens: nn.Embed - dtype: jnp.dtype = jnp.float32 # the dtype of the computation - - def setup(self): - self.dropout_layer = nn.Dropout(rate=self.config.dropout) - - embed_dim = self.config.d_model - self.padding_idx = self.config.pad_token_id - self.max_source_positions = self.config.max_position_embeddings - self.embed_scale = math.sqrt(embed_dim) if self.config.scale_embedding else 1.0 - - # OPT is set up so that if padding_idx is specified then offset the embedding ids by 2 - # and adjust num_embeddings appropriately. Other models don't have this hack - self.offset = 2 - self.embed_positions = nn.Embed( - self.config.max_position_embeddings + self.offset, - embed_dim, - embedding_init=jax.nn.initializers.normal(self.config.init_std), - ) - self.layers = FlaxOPTEncoderLayerCollection(self.config, self.dtype) - self.layernorm_embedding = nn.LayerNorm(dtype=self.dtype, epsilon=1e-05) - - def __call__( - self, - input_ids, - attention_mask, - position_ids, - output_attentions: bool = False, - output_hidden_states: bool = False, - return_dict: bool = True, - deterministic: bool = True, - ): - input_shape = input_ids.shape - input_ids = input_ids.reshape(-1, input_shape[-1]) - - inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale - - embed_pos = self.embed_positions(position_ids + self.offset) - - hidden_states = inputs_embeds + embed_pos - hidden_states = self.layernorm_embedding(hidden_states) - hidden_states = self.dropout_layer(hidden_states, deterministic=deterministic) - - outputs = self.layers( - hidden_states, - attention_mask, - deterministic=deterministic, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - ) - - if not return_dict: - return outputs - - return FlaxBaseModelOutput( - last_hidden_state=outputs.last_hidden_state, - hidden_states=outputs.hidden_states, - attentions=outputs.attentions, - ) - - # Copied from transformers.models.bart.modeling_flax_bart.FlaxBartDecoder with Bart->OPT class FlaxOPTDecoder(nn.Module): config: OPTConfig @@ -843,62 +666,57 @@ def setup(self): self.config.d_model, embedding_init=jax.nn.initializers.normal(self.config.init_std), ) - - self.encoder = FlaxOPTEncoder(self.config, dtype=self.dtype, embed_tokens=self.shared) - self.decoder = FlaxOPTDecoder(self.config, dtype=self.dtype, embed_tokens=self.shared) - - def _get_encoder_module(self): - return self.encoder - - def _get_decoder_module(self): - return self.decoder + self.transformer = FlaxOPTDecoder(self.config, dtype=self.dtype, embed_tokens=self.shared) + self.lm_head = nn.Dense( + self.config.vocab_size, + use_bias=False, + dtype=self.dtype, + kernel_init=jax.nn.initializers.normal(stddev=self.config.initializer_range), + ) def __call__( self, input_ids, attention_mask, - decoder_input_ids, - decoder_attention_mask, position_ids, - decoder_position_ids, + encoder_hidden_states: Optional[jnp.ndarray] = None, + encoder_attention_mask: Optional[jnp.ndarray] = None, output_attentions: bool = False, output_hidden_states: bool = False, return_dict: bool = True, deterministic: bool = True, + init_cache: bool = False, ): - encoder_outputs = self.encoder( - input_ids=input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - deterministic=deterministic, - ) - decoder_outputs = self.decoder( - input_ids=decoder_input_ids, - attention_mask=decoder_attention_mask, - position_ids=decoder_position_ids, - encoder_hidden_states=encoder_outputs[0], - encoder_attention_mask=attention_mask, + outputs = self.transformer( + input_ids, + attention_mask, + position_ids, + encoder_hidden_states, + encoder_attention_mask, + deterministic=deterministic, + init_cache=init_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, - deterministic=deterministic, ) + hidden_states = outputs[0] + + if self.config.tie_word_embeddings: + shared_kernel = self.transformer.variables["params"]["wte"]["embedding"].T + lm_logits = self.lm_head.apply({"params": {"kernel": shared_kernel}}, hidden_states) + else: + lm_logits = self.lm_head(hidden_states) + if not return_dict: - return decoder_outputs + encoder_outputs - - return FlaxSeq2SeqModelOutput( - last_hidden_state=decoder_outputs.last_hidden_state, - decoder_hidden_states=decoder_outputs.hidden_states, - decoder_attentions=decoder_outputs.attentions, - cross_attentions=decoder_outputs.cross_attentions, - encoder_last_hidden_state=encoder_outputs.last_hidden_state, - encoder_hidden_states=encoder_outputs.hidden_states, - encoder_attentions=encoder_outputs.attentions, + return (lm_logits,) + outputs[1:] + + return FlaxCausalLMOutputWithCrossAttentions( + logits=lm_logits, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + cross_attentions=outputs.cross_attentions, ) From abd1f3c7ef79e8dfa9762831a7456a1dbb0047ad Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Mon, 9 May 2022 10:38:17 +0200 Subject: [PATCH 051/145] add new version --- src/transformers/models/opt/modeling_opt.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 41e7e695bb779..7dfb00a57977f 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -827,7 +827,9 @@ def custom_forward(*inputs): ) if self.layer_norm: - layer_outputs = self.layer_norm(layer_outputs) + new_layer_output = (self.layer_norm(layer_outputs[0]),) + new_layer_output += (layer_outputs[1:],) + layer_outputs = new_layer_output hidden_states = layer_outputs[0] if use_cache: @@ -975,7 +977,11 @@ def __init__(self, config): super().__init__(config) self.model = OPTDecoderWrapper(config) - self.lm_head = self.model.decoder.output_projection + if config.output_projection: + self.lm_head = self.model.decoder.output_projection + else: + self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False) + self.lm_head.weight = nn.Parameter(self.model.decoder.embed_tokens.weight.t()) # Initialize weights and apply final processing self.post_init() From 5c5c85866f1540531f1500322e8ccda743965b8f Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Mon, 9 May 2022 10:45:35 +0200 Subject: [PATCH 052/145] causal LM output --- src/transformers/models/opt/modeling_opt.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 41e7e695bb779..8553a6bfc5d46 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -28,7 +28,7 @@ from ...modeling_outputs import ( BaseModelOutputWithPastAndCrossAttentions, CausalLMOutputWithCrossAttentions, - Seq2SeqModelOutput, + CausalLMOutput, ) from ...modeling_utils import PreTrainedModel from ...utils import ( @@ -884,7 +884,7 @@ def get_decoder(self): @add_code_sample_docstrings( processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, - output_type=Seq2SeqModelOutput, + output_type=CausalLMOutput, config_class=_CONFIG_FOR_DOC, expected_output=_EXPECTED_OUTPUT_SHAPE, ) @@ -904,7 +904,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, Seq2SeqModelOutput]: + ) -> Union[Tuple, CausalLMOutput]: # different to other models, OPT automatically creates decoder_input_ids from # input_ids if no decoder_input_ids are provided @@ -927,7 +927,7 @@ def forward( use_cache = use_cache if use_cache is not None else self.config.use_cache return_dict = return_dict if return_dict is not None else self.config.use_return_dict - # If the user passed a tuple for encoder_outputs, we wrap it in a BaseModelOutput when return_dict=True + # If the user passed a tuple for encoder_outputs, we wrap it in a CausalLMOutput when return_dict=True # decoder outputs consists of (dec_features, past_key_value, dec_hidden, dec_attn) decoder_outputs = self.decoder( @@ -943,7 +943,7 @@ def forward( return_dict=return_dict, ) - return Seq2SeqModelOutput( + return CausalLMOutput( last_hidden_state=decoder_outputs.last_hidden_state, past_key_values=decoder_outputs.past_key_values, decoder_hidden_states=decoder_outputs.hidden_states, From 27b55c98fdc091c432b7207c425fef1fd7092038 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Mon, 9 May 2022 11:09:19 +0200 Subject: [PATCH 053/145] replace output type with BaseModelOutputWithPastAndCrossAttentions --- src/transformers/models/opt/modeling_opt.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 499d66f663206..f0080cac3f6cb 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -28,7 +28,6 @@ from ...modeling_outputs import ( BaseModelOutputWithPastAndCrossAttentions, CausalLMOutputWithCrossAttentions, - CausalLMOutput, ) from ...modeling_utils import PreTrainedModel from ...utils import ( @@ -886,7 +885,7 @@ def get_decoder(self): @add_code_sample_docstrings( processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, - output_type=CausalLMOutput, + output_type=BaseModelOutputWithPastAndCrossAttentions, config_class=_CONFIG_FOR_DOC, expected_output=_EXPECTED_OUTPUT_SHAPE, ) @@ -906,7 +905,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, CausalLMOutput]: + ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]: # different to other models, OPT automatically creates decoder_input_ids from # input_ids if no decoder_input_ids are provided @@ -929,7 +928,7 @@ def forward( use_cache = use_cache if use_cache is not None else self.config.use_cache return_dict = return_dict if return_dict is not None else self.config.use_return_dict - # If the user passed a tuple for encoder_outputs, we wrap it in a CausalLMOutput when return_dict=True + # If the user passed a tuple for encoder_outputs, we wrap it in a BaseModelOutputWithPastAndCrossAttentions when return_dict=True # decoder outputs consists of (dec_features, past_key_value, dec_hidden, dec_attn) decoder_outputs = self.decoder( @@ -945,11 +944,11 @@ def forward( return_dict=return_dict, ) - return CausalLMOutput( + return BaseModelOutputWithPastAndCrossAttentions( last_hidden_state=decoder_outputs.last_hidden_state, past_key_values=decoder_outputs.past_key_values, - decoder_hidden_states=decoder_outputs.hidden_states, - decoder_attentions=decoder_outputs.attentions, + hidden_states=decoder_outputs.hidden_states, + attentions=decoder_outputs.attentions, cross_attentions=decoder_outputs.cross_attentions, ) From aebd19e3e53e964c5ae3b2608821c2db8c55c060 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Mon, 9 May 2022 11:53:35 +0200 Subject: [PATCH 054/145] revert working config from 150m to 350m --- src/transformers/models/opt/configuration_opt.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/transformers/models/opt/configuration_opt.py b/src/transformers/models/opt/configuration_opt.py index e5648570ce847..0cbb9021db661 100644 --- a/src/transformers/models/opt/configuration_opt.py +++ b/src/transformers/models/opt/configuration_opt.py @@ -101,13 +101,13 @@ def __init__( self, vocab_size=50272, # TODO check the real value max_position_embeddings=2048, - num_hidden_layers=12, - num_attention_heads=12, - ffn_dim=3072, + num_hidden_layers=24, + num_attention_heads=16, + ffn_dim=4096, layerdrop=0.0, activation_function="gelu", - d_model=768, - embed_dim=768, + d_model=1024, + embed_dim=512, dropout=0.1, attention_dropout=0.0, activation_dropout=0.0, @@ -120,8 +120,8 @@ def __init__( eos_token_id=2, decoder_start_token_id=2, forced_eos_token_id=2, - output_projection=False, - decoder_layernorm=True, + output_projection=True, + decoder_layernorm=False, **kwargs ): self.vocab_size = vocab_size From d0723aa684861d76b8459009513652302862496d Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Mon, 9 May 2022 11:53:49 +0200 Subject: [PATCH 055/145] clean --- src/transformers/models/opt/modeling_opt.py | 2 +- tests/models/opt/test_modeling_opt.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index f0080cac3f6cb..273fd42be6c94 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -943,7 +943,7 @@ def forward( output_hidden_states=output_hidden_states, return_dict=return_dict, ) - + return BaseModelOutputWithPastAndCrossAttentions( last_hidden_state=decoder_outputs.last_hidden_state, past_key_values=decoder_outputs.past_key_values, diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index 208efedd5f859..2e7c7117765c1 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -787,9 +787,8 @@ def test_retain_grad_hidden_states_attentions(self): class OPTEmbeddingsTest(unittest.TestCase): def setUp(self): super().setUp() - self.path_model = "ArthurZ/opt-350m" # TODO add the model on the hub - # self.path_logits_meta = "/home/younes/Desktop/Work/metaseq-conversion/logits_metaseq_gpt2_tokenizer.p" # TODO add the logits somewhere? - + self.path_model = "ArthurZ/opt-350m" + def test_load_model(self): try: _ = OPTForCausalLM.from_pretrained(self.path_model) @@ -859,3 +858,4 @@ def test_generation(self): predicted_next_token = gen(prompt)[0]["generated_token_ids"][len_input_sentence] GEN_OUTPUT.append(predicted_next_token) self.assertListEqual(GEN_OUTPUT, NEXT_TOKENS) + From 757574903aba72f640f2ad6ef95ef3b537cbfa82 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Mon, 9 May 2022 12:01:36 +0200 Subject: [PATCH 056/145] removed decoder input ids --- tests/models/opt/test_modeling_opt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index 2e7c7117765c1..208e440ea7fbc 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -57,11 +57,11 @@ def prepare_opt_inputs_dict( cross_attn_head_mask = torch.ones(config.num_hidden_layers, config.num_attention_heads, device=torch_device) return { "input_ids": input_ids, - "decoder_input_ids": decoder_input_ids, + # "decoder_input_ids": decoder_input_ids, "attention_mask": attention_mask, - "decoder_attention_mask": attention_mask, + # "decoder_attention_mask": attention_mask, "head_mask": head_mask, - "decoder_head_mask": decoder_head_mask, + # "decoder_head_mask": decoder_head_mask, "cross_attn_head_mask": cross_attn_head_mask, } From 66e829876d156373277e4e63a3c848ebbf7895f9 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Mon, 9 May 2022 12:24:21 +0200 Subject: [PATCH 057/145] fixed embed dim --- tests/models/opt/test_modeling_opt.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index 208e440ea7fbc..eaca668b9c42b 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -86,6 +86,7 @@ def __init__( eos_token_id=2, pad_token_id=1, bos_token_id=0, + embed_dim=16, ): self.parent = parent self.batch_size = batch_size @@ -104,6 +105,7 @@ def __init__( self.eos_token_id = eos_token_id self.pad_token_id = pad_token_id self.bos_token_id = bos_token_id + self.embed_dim = embed_dim self.is_encoder_decoder = False def prepare_config_and_inputs(self): @@ -132,6 +134,7 @@ def get_config(self): eos_token_id=self.eos_token_id, bos_token_id=self.bos_token_id, pad_token_id=self.pad_token_id, + embed_dim=self.embed_dim, ) def get_pipeline_config(self): From 8d4920ebc83beb175124bf84bc5167ecb71a4dde Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Mon, 9 May 2022 12:28:42 +0200 Subject: [PATCH 058/145] more embed_dim issues --- tests/models/opt/test_modeling_opt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index eaca668b9c42b..297812c8efc11 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -563,6 +563,7 @@ def __init__( decoder_attention_heads=4, max_position_embeddings=30, is_encoder_decoder=False, + embed_dim=16, pad_token_id=0, bos_token_id=1, eos_token_id=2, @@ -593,7 +594,7 @@ def __init__( self.use_cache = use_cache self.max_position_embeddings = max_position_embeddings self.is_encoder_decoder = is_encoder_decoder - + self.embed_dim=embed_dim self.scope = None self.decoder_key_length = decoder_seq_length self.base_model_out_len = 2 @@ -625,6 +626,7 @@ def prepare_config_and_inputs(self): decoder_start_token_id=self.decoder_start_token_id, max_position_embeddings=self.max_position_embeddings, is_encoder_decoder=self.is_encoder_decoder, + embed_dim=self.embed_dim ) return ( From c0058401fa91ad55f3661ccfbe1bb9e9a046cefa Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Mon, 9 May 2022 12:37:50 +0200 Subject: [PATCH 059/145] make style + removed enc_dec test --- src/transformers/models/opt/configuration_opt.py | 1 - src/transformers/models/opt/modeling_opt.py | 9 ++------- tests/models/opt/test_modeling_opt.py | 11 +++-------- 3 files changed, 5 insertions(+), 16 deletions(-) diff --git a/src/transformers/models/opt/configuration_opt.py b/src/transformers/models/opt/configuration_opt.py index 0cbb9021db661..79bfe93c7c72f 100644 --- a/src/transformers/models/opt/configuration_opt.py +++ b/src/transformers/models/opt/configuration_opt.py @@ -142,7 +142,6 @@ def __init__( self.scale_embedding = scale_embedding # scale factor will be sqrt(d_model) if True self.output_projection = output_projection self.decoder_layernorm = decoder_layernorm - super().__init__( pad_token_id=pad_token_id, bos_token_id=bos_token_id, diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 273fd42be6c94..b38a958c71424 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -25,10 +25,7 @@ from torch.nn import CrossEntropyLoss from ...activations import ACT2FN -from ...modeling_outputs import ( - BaseModelOutputWithPastAndCrossAttentions, - CausalLMOutputWithCrossAttentions, -) +from ...modeling_outputs import BaseModelOutputWithPastAndCrossAttentions, CausalLMOutputWithCrossAttentions from ...modeling_utils import PreTrainedModel from ...utils import ( add_code_sample_docstrings, @@ -444,8 +441,6 @@ def forward( return outputs - - OPT_START_DOCSTRING = r""" This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads @@ -943,7 +938,7 @@ def forward( output_hidden_states=output_hidden_states, return_dict=return_dict, ) - + return BaseModelOutputWithPastAndCrossAttentions( last_hidden_state=decoder_outputs.last_hidden_state, past_key_values=decoder_outputs.past_key_values, diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index 297812c8efc11..31db0d456c462 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -339,10 +339,6 @@ def test_decoder_model_past_with_large_inputs(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_decoder_model_past_large_inputs(*config_and_inputs) - def test_encoder_decoder_model_standalone(self): - config_and_inputs = self.model_tester.prepare_config_and_inputs_for_common() - self.model_tester.check_encoder_decoder_model_standalone(*config_and_inputs) - # OPTForSequenceClassification does not support inputs_embeds def test_inputs_embeds(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -594,7 +590,7 @@ def __init__( self.use_cache = use_cache self.max_position_embeddings = max_position_embeddings self.is_encoder_decoder = is_encoder_decoder - self.embed_dim=embed_dim + self.embed_dim = embed_dim self.scope = None self.decoder_key_length = decoder_seq_length self.base_model_out_len = 2 @@ -626,7 +622,7 @@ def prepare_config_and_inputs(self): decoder_start_token_id=self.decoder_start_token_id, max_position_embeddings=self.max_position_embeddings, is_encoder_decoder=self.is_encoder_decoder, - embed_dim=self.embed_dim + embed_dim=self.embed_dim, ) return ( @@ -793,7 +789,7 @@ class OPTEmbeddingsTest(unittest.TestCase): def setUp(self): super().setUp() self.path_model = "ArthurZ/opt-350m" - + def test_load_model(self): try: _ = OPTForCausalLM.from_pretrained(self.path_model) @@ -863,4 +859,3 @@ def test_generation(self): predicted_next_token = gen(prompt)[0]["generated_token_ids"][len_input_sentence] GEN_OUTPUT.append(predicted_next_token) self.assertListEqual(GEN_OUTPUT, NEXT_TOKENS) - From 84eb49761d7b8f064baf7036178c254f10eb00c3 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Mon, 9 May 2022 13:52:05 +0200 Subject: [PATCH 060/145] update falx model --- .../models/opt/modeling_flax_opt.py | 61 ++++++++----------- 1 file changed, 26 insertions(+), 35 deletions(-) diff --git a/src/transformers/models/opt/modeling_flax_opt.py b/src/transformers/models/opt/modeling_flax_opt.py index 42ae856f7f104..21bc4f127a66e 100644 --- a/src/transformers/models/opt/modeling_flax_opt.py +++ b/src/transformers/models/opt/modeling_flax_opt.py @@ -223,7 +223,6 @@ def shift_tokens_right(input_ids: np.array, pad_token_id: int, decoder_start_tok return shifted_input_ids -# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartAttention with Bart->OPT class FlaxOPTAttention(nn.Module): config: OPTConfig embed_dim: int @@ -313,7 +312,7 @@ def __call__( batch_size = hidden_states.shape[0] # get query proj - query_states = self.q_proj(hidden_states) * self.scaling + query_states = self.q_proj(hidden_states) # get key, value proj if is_cross_attention: # cross_attentions @@ -666,57 +665,49 @@ def setup(self): self.config.d_model, embedding_init=jax.nn.initializers.normal(self.config.init_std), ) - self.transformer = FlaxOPTDecoder(self.config, dtype=self.dtype, embed_tokens=self.shared) - self.lm_head = nn.Dense( - self.config.vocab_size, - use_bias=False, - dtype=self.dtype, - kernel_init=jax.nn.initializers.normal(stddev=self.config.initializer_range), - ) + + self.decoder = FlaxOPTDecoder(self.config, dtype=self.dtype, embed_tokens=self.shared) + + def _get_decoder_module(self): + return self.decoder def __call__( self, input_ids, attention_mask, + decoder_input_ids, + decoder_attention_mask, position_ids, - encoder_hidden_states: Optional[jnp.ndarray] = None, - encoder_attention_mask: Optional[jnp.ndarray] = None, + decoder_position_ids, output_attentions: bool = False, output_hidden_states: bool = False, return_dict: bool = True, deterministic: bool = True, - init_cache: bool = False, ): - outputs = self.transformer( - input_ids, - attention_mask, - position_ids, - encoder_hidden_states, - encoder_attention_mask, - deterministic=deterministic, - init_cache=init_cache, + decoder_outputs = self.decoder( + input_ids=decoder_input_ids, + attention_mask=decoder_attention_mask, + position_ids=decoder_position_ids, + # encoder_hidden_states=encoder_outputs[0], + encoder_attention_mask=attention_mask, output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, + deterministic=deterministic, ) - hidden_states = outputs[0] - - if self.config.tie_word_embeddings: - shared_kernel = self.transformer.variables["params"]["wte"]["embedding"].T - lm_logits = self.lm_head.apply({"params": {"kernel": shared_kernel}}, hidden_states) - else: - lm_logits = self.lm_head(hidden_states) - if not return_dict: - return (lm_logits,) + outputs[1:] - - return FlaxCausalLMOutputWithCrossAttentions( - logits=lm_logits, - hidden_states=outputs.hidden_states, - attentions=outputs.attentions, - cross_attentions=outputs.cross_attentions, + return decoder_outputs + + return FlaxSeq2SeqModelOutput( + last_hidden_state=decoder_outputs.last_hidden_state, + decoder_hidden_states=decoder_outputs.hidden_states, + decoder_attentions=decoder_outputs.attentions, + cross_attentions=decoder_outputs.cross_attentions, + # encoder_last_hidden_state=encoder_outputs.last_hidden_state, + # encoder_hidden_states=encoder_outputs.hidden_states, + # encoder_attentions=encoder_outputs.attentions, ) From 043a109785c3bd2cad5ed75fe3a9cee8f8cf529b Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Mon, 9 May 2022 14:04:07 +0200 Subject: [PATCH 061/145] removed troublesome copy --- .../models/opt/modeling_flax_opt.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/src/transformers/models/opt/modeling_flax_opt.py b/src/transformers/models/opt/modeling_flax_opt.py index 21bc4f127a66e..e5ff01a5f0b83 100644 --- a/src/transformers/models/opt/modeling_flax_opt.py +++ b/src/transformers/models/opt/modeling_flax_opt.py @@ -653,8 +653,6 @@ def __call__( cross_attentions=outputs.cross_attentions, ) - -# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartModule with Bart->OPT class FlaxOPTModule(nn.Module): config: OPTConfig dtype: jnp.dtype = jnp.float32 # the dtype of the computation @@ -675,8 +673,6 @@ def __call__( self, input_ids, attention_mask, - decoder_input_ids, - decoder_attention_mask, position_ids, decoder_position_ids, output_attentions: bool = False, @@ -686,11 +682,9 @@ def __call__( ): decoder_outputs = self.decoder( - input_ids=decoder_input_ids, - attention_mask=decoder_attention_mask, - position_ids=decoder_position_ids, - # encoder_hidden_states=encoder_outputs[0], - encoder_attention_mask=attention_mask, + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, @@ -700,14 +694,11 @@ def __call__( if not return_dict: return decoder_outputs - return FlaxSeq2SeqModelOutput( + return FlaxSeq2SeqModelOutput( # TODO change model output last_hidden_state=decoder_outputs.last_hidden_state, decoder_hidden_states=decoder_outputs.hidden_states, decoder_attentions=decoder_outputs.attentions, cross_attentions=decoder_outputs.cross_attentions, - # encoder_last_hidden_state=encoder_outputs.last_hidden_state, - # encoder_hidden_states=encoder_outputs.hidden_states, - # encoder_attentions=encoder_outputs.attentions, ) From 8ba7cbcafe39cdb1d3f7ed8b0882307f284fb7d0 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Mon, 9 May 2022 14:06:55 +0200 Subject: [PATCH 062/145] added is_encoder_decoder=False to config --- tests/models/opt/test_modeling_opt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index 31db0d456c462..34042f27e9e02 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -135,6 +135,7 @@ def get_config(self): bos_token_id=self.bos_token_id, pad_token_id=self.pad_token_id, embed_dim=self.embed_dim, + is_encoder_decoder=False ) def get_pipeline_config(self): From 2099b5f580556eb91d9a984d0ce16decb9296062 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Mon, 9 May 2022 14:25:40 +0200 Subject: [PATCH 063/145] added set_input emb fuinction to model class --- src/transformers/models/opt/modeling_opt.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index b38a958c71424..b3a2af8a8cf16 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -872,6 +872,9 @@ def __init__(self, config: OPTConfig): def get_input_embeddings(self): return self.decoder.embed_tokens + + def set_input_embeddings(self, value): + self.decoder.embed_tokens = value def get_decoder(self): return self.decoder From 1c9580fc9232c2caacfbc45ff744fb8628546a68 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Mon, 9 May 2022 14:40:37 +0200 Subject: [PATCH 064/145] requires torch on embed test --- tests/models/opt/test_modeling_opt.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index 34042f27e9e02..c43c98c8c5d91 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -135,7 +135,7 @@ def get_config(self): bos_token_id=self.bos_token_id, pad_token_id=self.pad_token_id, embed_dim=self.embed_dim, - is_encoder_decoder=False + is_encoder_decoder=False, ) def get_pipeline_config(self): @@ -539,7 +539,7 @@ def test_inference_no_head(self): # config_params = {k: getattr(config, k, "MISSING") for k, v in expected_params.items()} # self.assertDictEqual(expected_params, config_params) - +@require_torch class OPTStandaloneDecoderModelTester: def __init__( self, @@ -796,7 +796,8 @@ def test_load_model(self): _ = OPTForCausalLM.from_pretrained(self.path_model) except BaseException: self.fail("Failed loading model") - + + @require_torch @torch.no_grad() def test_logits(self): model = OPTForCausalLM.from_pretrained(self.path_model) From 9f6291d3003d7a3f18e1e19ad527987b5707cdf3 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Mon, 9 May 2022 14:50:17 +0200 Subject: [PATCH 065/145] use head mask instead of decoder head mask input param solves a test --- src/transformers/models/opt/modeling_opt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index b3a2af8a8cf16..2da75a568c8ca 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -931,11 +931,11 @@ def forward( # decoder outputs consists of (dec_features, past_key_value, dec_hidden, dec_attn) decoder_outputs = self.decoder( input_ids=decoder_input_ids, - attention_mask=decoder_attention_mask, - head_mask=decoder_head_mask, + attention_mask=attention_mask, + head_mask=head_mask, cross_attn_head_mask=cross_attn_head_mask, past_key_values=past_key_values, - inputs_embeds=decoder_inputs_embeds, + inputs_embeds=inputs_embeds, use_cache=use_cache, output_attentions=output_attentions, output_hidden_states=output_hidden_states, From 740fcf506827a5e1851f4cee226d474c52fb599a Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Mon, 9 May 2022 15:48:44 +0200 Subject: [PATCH 066/145] 8 test remaining, update --- tests/models/opt/test_modeling_opt.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index c43c98c8c5d91..61c010c682d8d 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -218,6 +218,7 @@ def _get_config_and_data(self): eos_token_id=2, pad_token_id=1, bos_token_id=0, + embed_dim=24 ) return config, input_ids, batch_size @@ -344,7 +345,7 @@ def test_decoder_model_past_with_large_inputs(self): def test_inputs_embeds(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - for model_class in OPTModel: + for model_class in (OPTModel,): model = model_class(config) model.to(torch_device) model.eval() @@ -366,7 +367,7 @@ def test_inputs_embeds(self): else: inputs["inputs_embeds"] = wte(encoder_input_ids) inputs["decoder_inputs_embeds"] = wte(decoder_input_ids) - + with torch.no_grad(): model(**inputs)[0] @@ -555,7 +556,7 @@ def __init__( use_labels=True, decoder_start_token_id=2, decoder_ffn_dim=32, - decoder_layers=4, + decoder_layers=24, encoder_attention_heads=4, decoder_attention_heads=4, max_position_embeddings=30, @@ -581,8 +582,7 @@ def __init__( self.num_hidden_layers = decoder_layers self.decoder_layers = decoder_layers self.decoder_ffn_dim = decoder_ffn_dim - self.encoder_attention_heads = encoder_attention_heads - self.decoder_attention_heads = decoder_attention_heads + self.num_attention_heads = decoder_attention_heads self.eos_token_id = eos_token_id self.bos_token_id = bos_token_id @@ -611,11 +611,9 @@ def prepare_config_and_inputs(self): config = OPTConfig( vocab_size=self.vocab_size, d_model=self.d_model, - encoder_layers=self.decoder_layers, - decoder_layers=self.decoder_layers, - decoder_ffn_dim=self.decoder_ffn_dim, - encoder_attention_heads=self.encoder_attention_heads, - decoder_attention_heads=self.decoder_attention_heads, + num_hidden_layers=self.num_hidden_layers, + ffn_dim=self.decoder_ffn_dim, + num_attention_heads=self.num_attention_heads, eos_token_id=self.eos_token_id, bos_token_id=self.bos_token_id, use_cache=self.use_cache, From f8c276b5fa30be709b26e5da8d4232d9ccb2605f Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Mon, 9 May 2022 16:12:40 +0200 Subject: [PATCH 067/145] Updated create_and_check_decoder_model_past_large_inputs --- tests/models/opt/test_modeling_opt.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index 61c010c682d8d..ef012e6803cf2 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -148,7 +148,9 @@ def prepare_config_and_inputs_for_common(self): return config, inputs_dict def create_and_check_decoder_model_past_large_inputs(self, config, inputs_dict): - model = OPTModel(config=config).get_decoder().to(torch_device).eval() + model = OPTModel(config=config).to(torch_device).eval() + # previousdly OPTModel(config=config).get_decoder().to(torch_device).eval() + # tried OPTForCausalkML input_ids = inputs_dict["input_ids"] attention_mask = inputs_dict["attention_mask"] head_mask = inputs_dict["head_mask"] @@ -156,7 +158,7 @@ def create_and_check_decoder_model_past_large_inputs(self, config, inputs_dict): # first forward pass outputs = model(input_ids, attention_mask=attention_mask, head_mask=head_mask, use_cache=True) - output, past_key_values = outputs.to_tuple() + output, past_key_values = outputs.to_tuple() # TODO, TOO MANY VALUES TO UNPACK HERE # create hypothetical multiple next token and extent to next_input_ids next_tokens = ids_tensor((self.batch_size, 3), config.vocab_size) From fff035f543b751a3a1e4c44a68af2b2d371d2427 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Mon, 9 May 2022 16:13:31 +0200 Subject: [PATCH 068/145] Make style --- src/transformers/models/opt/modeling_flax_opt.py | 3 ++- src/transformers/models/opt/modeling_opt.py | 2 +- tests/models/opt/test_modeling_opt.py | 13 +++++++------ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/transformers/models/opt/modeling_flax_opt.py b/src/transformers/models/opt/modeling_flax_opt.py index e5ff01a5f0b83..7f4f497e3678b 100644 --- a/src/transformers/models/opt/modeling_flax_opt.py +++ b/src/transformers/models/opt/modeling_flax_opt.py @@ -653,6 +653,7 @@ def __call__( cross_attentions=outputs.cross_attentions, ) + class FlaxOPTModule(nn.Module): config: OPTConfig dtype: jnp.dtype = jnp.float32 # the dtype of the computation @@ -694,7 +695,7 @@ def __call__( if not return_dict: return decoder_outputs - return FlaxSeq2SeqModelOutput( # TODO change model output + return FlaxSeq2SeqModelOutput( # TODO change model output last_hidden_state=decoder_outputs.last_hidden_state, decoder_hidden_states=decoder_outputs.hidden_states, decoder_attentions=decoder_outputs.attentions, diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 2da75a568c8ca..c9064617911ff 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -872,7 +872,7 @@ def __init__(self, config: OPTConfig): def get_input_embeddings(self): return self.decoder.embed_tokens - + def set_input_embeddings(self, value): self.decoder.embed_tokens = value diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index ef012e6803cf2..c7c745ea4f1d1 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -148,7 +148,7 @@ def prepare_config_and_inputs_for_common(self): return config, inputs_dict def create_and_check_decoder_model_past_large_inputs(self, config, inputs_dict): - model = OPTModel(config=config).to(torch_device).eval() + model = OPTModel(config=config).to(torch_device).eval() # previousdly OPTModel(config=config).get_decoder().to(torch_device).eval() # tried OPTForCausalkML input_ids = inputs_dict["input_ids"] @@ -158,7 +158,7 @@ def create_and_check_decoder_model_past_large_inputs(self, config, inputs_dict): # first forward pass outputs = model(input_ids, attention_mask=attention_mask, head_mask=head_mask, use_cache=True) - output, past_key_values = outputs.to_tuple() # TODO, TOO MANY VALUES TO UNPACK HERE + output, past_key_values = outputs.to_tuple() # TODO, TOO MANY VALUES TO UNPACK HERE # create hypothetical multiple next token and extent to next_input_ids next_tokens = ids_tensor((self.batch_size, 3), config.vocab_size) @@ -220,7 +220,7 @@ def _get_config_and_data(self): eos_token_id=2, pad_token_id=1, bos_token_id=0, - embed_dim=24 + embed_dim=24, ) return config, input_ids, batch_size @@ -369,7 +369,7 @@ def test_inputs_embeds(self): else: inputs["inputs_embeds"] = wte(encoder_input_ids) inputs["decoder_inputs_embeds"] = wte(decoder_input_ids) - + with torch.no_grad(): model(**inputs)[0] @@ -542,6 +542,7 @@ def test_inference_no_head(self): # config_params = {k: getattr(config, k, "MISSING") for k, v in expected_params.items()} # self.assertDictEqual(expected_params, config_params) + @require_torch class OPTStandaloneDecoderModelTester: def __init__( @@ -584,7 +585,7 @@ def __init__( self.num_hidden_layers = decoder_layers self.decoder_layers = decoder_layers self.decoder_ffn_dim = decoder_ffn_dim - + self.num_attention_heads = decoder_attention_heads self.eos_token_id = eos_token_id self.bos_token_id = bos_token_id @@ -796,7 +797,7 @@ def test_load_model(self): _ = OPTForCausalLM.from_pretrained(self.path_model) except BaseException: self.fail("Failed loading model") - + @require_torch @torch.no_grad() def test_logits(self): From 30ed9f6a8e6c49d3782ef78aa79a5c4f68c1b131 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Mon, 9 May 2022 16:29:03 +0200 Subject: [PATCH 069/145] update op tokenizer with condition --- src/transformers/models/auto/tokenization_auto.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index df80135553ca7..ffff0971dbc34 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -107,7 +107,7 @@ ("blenderbot", ("BlenderbotTokenizer", "BlenderbotTokenizerFast")), ("tapex", ("TapexTokenizer", None)), ("bart", ("BartTokenizer", "BartTokenizerFast")), - ("opt", ("GPT2Tokenizer", "GPT2TokenizerFast")), + ("opt", ("GPT2Tokenizer", "GPT2TokenizerFast" if is_tokenizers_available() else None)), ("longformer", ("LongformerTokenizer", "LongformerTokenizerFast" if is_tokenizers_available() else None)), ("roberta", ("RobertaTokenizer", "RobertaTokenizerFast" if is_tokenizers_available() else None)), ( From 69c7ae6df43ba244d2683e88e1aa977ee97ad05b Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Mon, 9 May 2022 16:38:51 +0200 Subject: [PATCH 070/145] make style --- src/transformers/models/auto/tokenization_auto.py | 2 +- tests/models/opt/test_modeling_opt.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index ffff0971dbc34..573e908268a5a 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -107,7 +107,7 @@ ("blenderbot", ("BlenderbotTokenizer", "BlenderbotTokenizerFast")), ("tapex", ("TapexTokenizer", None)), ("bart", ("BartTokenizer", "BartTokenizerFast")), - ("opt", ("GPT2Tokenizer", "GPT2TokenizerFast" if is_tokenizers_available() else None)), + ("opt", ("GPT2Tokenizer", "GPT2TokenizerFast" if is_tokenizers_available() else None)), ("longformer", ("LongformerTokenizer", "LongformerTokenizerFast" if is_tokenizers_available() else None)), ("roberta", ("RobertaTokenizer", "RobertaTokenizerFast" if is_tokenizers_available() else None)), ( diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index c7c745ea4f1d1..c23b853ad71dd 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -785,8 +785,8 @@ def test_retain_grad_hidden_states_attentions(self): # TODO solve the @torch.no_grad() issue -@require_torch @require_tokenizers +@require_torch class OPTEmbeddingsTest(unittest.TestCase): def setUp(self): super().setUp() @@ -828,6 +828,7 @@ def test_logits(self): assert torch.allclose(logits, logits_meta, atol=1e-4) +@require_tokenizers class OPTGenerationTest(unittest.TestCase): def setUp(self): super().setUp() From ff09958a577a7bc5aec760aca7d8b804133d8df8 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Tue, 10 May 2022 13:40:21 +0200 Subject: [PATCH 071/145] See if I can push --- src/transformers/models/opt/modeling_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index c9064617911ff..7c50b21d13fcb 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -48,7 +48,7 @@ OPT_PRETRAINED_MODEL_ARCHIVE_LIST = [ - "opt350-m", + "facebook/opt350m", # See all OPT models at https://huggingface.co/models?filter=opt ] From 0555b9222eb37dcdccea1225eae9bda9ac1817af Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Tue, 10 May 2022 14:07:29 +0200 Subject: [PATCH 072/145] some clean up --- src/transformers/models/opt/modeling_opt.py | 79 +--- tests/models/opt/test_modeling_opt.py | 397 +------------------- 2 files changed, 25 insertions(+), 451 deletions(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 7c50b21d13fcb..8b43c73a42875 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -25,7 +25,7 @@ from torch.nn import CrossEntropyLoss from ...activations import ACT2FN -from ...modeling_outputs import BaseModelOutputWithPastAndCrossAttentions, CausalLMOutputWithCrossAttentions +from ...modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast from ...modeling_utils import PreTrainedModel from ...utils import ( add_code_sample_docstrings, @@ -53,22 +53,6 @@ ] -def shift_tokens_right(input_ids: torch.Tensor, pad_token_id: int, decoder_start_token_id: int): - """ - Shift input ids one token to the right. - """ - shifted_input_ids = input_ids.new_zeros(input_ids.shape) - shifted_input_ids[:, 1:] = input_ids[:, :-1].clone() - shifted_input_ids[:, 0] = decoder_start_token_id - - if pad_token_id is None: - raise ValueError("self.model.config.pad_token_id has to be defined.") - # replace possible -100 values in labels by `pad_token_id` - shifted_input_ids.masked_fill_(shifted_input_ids == -100, pad_token_id) - - return shifted_input_ids - - def _make_causal_mask(input_ids_shape: torch.Size, dtype: torch.dtype, past_key_values_length: int = 0): """ Make causal mask used for bi-directional self-attention. @@ -663,7 +647,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]: + ) -> Union[Tuple, BaseModelOutputWithPast]: r""" Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): @@ -846,12 +830,11 @@ def custom_forward(*inputs): for v in [hidden_states, next_cache, all_hidden_states, all_self_attns, all_cross_attentions] if v is not None ) - return BaseModelOutputWithPastAndCrossAttentions( + return BaseModelOutputWithPast( last_hidden_state=hidden_states, past_key_values=next_cache, hidden_states=all_hidden_states, attentions=all_self_attns, - cross_attentions=all_cross_attentions, ) @@ -863,9 +846,8 @@ class OPTModel(OPTPretrainedModel): def __init__(self, config: OPTConfig): super().__init__(config) - # padding_idx, vocab_size = config.pad_token_id, config.vocab_size - self.decoder = OPTDecoder(config) + # padding_idx, vocab_size = config.pad_token_id, config.vocab_size # Initialize weights and apply final processing self.post_init() @@ -883,7 +865,7 @@ def get_decoder(self): @add_code_sample_docstrings( processor_class=_TOKENIZER_FOR_DOC, checkpoint=_CHECKPOINT_FOR_DOC, - output_type=BaseModelOutputWithPastAndCrossAttentions, + output_type=BaseModelOutputWithPast, config_class=_CONFIG_FOR_DOC, expected_output=_EXPECTED_OUTPUT_SHAPE, ) @@ -891,33 +873,14 @@ def forward( self, input_ids: torch.LongTensor = None, attention_mask: Optional[torch.Tensor] = None, - decoder_input_ids: Optional[torch.LongTensor] = None, - decoder_attention_mask: Optional[torch.LongTensor] = None, head_mask: Optional[torch.Tensor] = None, - decoder_head_mask: Optional[torch.Tensor] = None, - cross_attn_head_mask: Optional[torch.Tensor] = None, past_key_values: Optional[List[torch.FloatTensor]] = None, inputs_embeds: Optional[torch.FloatTensor] = None, - decoder_inputs_embeds: Optional[torch.FloatTensor] = None, use_cache: Optional[bool] = None, output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, BaseModelOutputWithPastAndCrossAttentions]: - - # different to other models, OPT automatically creates decoder_input_ids from - # input_ids if no decoder_input_ids are provided - if decoder_input_ids is None and decoder_inputs_embeds is None: - if input_ids is None: - raise ValueError( - "If no `decoder_input_ids` or `decoder_inputs_embeds` are " - "passed, `input_ids` cannot be `None`. Please pass either " - "`input_ids` or `decoder_input_ids` or `decoder_inputs_embeds`." - ) - - decoder_input_ids = shift_tokens_right( - input_ids, self.config.pad_token_id, self.config.decoder_start_token_id - ) + ) -> Union[Tuple, BaseModelOutputWithPast]: output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -930,10 +893,9 @@ def forward( # decoder outputs consists of (dec_features, past_key_value, dec_hidden, dec_attn) decoder_outputs = self.decoder( - input_ids=decoder_input_ids, + input_ids=input_ids, attention_mask=attention_mask, head_mask=head_mask, - cross_attn_head_mask=cross_attn_head_mask, past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, @@ -942,37 +904,21 @@ def forward( return_dict=return_dict, ) - return BaseModelOutputWithPastAndCrossAttentions( + return BaseModelOutputWithPast( last_hidden_state=decoder_outputs.last_hidden_state, past_key_values=decoder_outputs.past_key_values, hidden_states=decoder_outputs.hidden_states, attentions=decoder_outputs.attentions, - cross_attentions=decoder_outputs.cross_attentions, ) -# Copied from transformers.models.bart.modeling_bart.BartDecoderWrapper with Bart->OPT -class OPTDecoderWrapper(OPTPretrainedModel): - """ - This wrapper class is a helper class to correctly load pretrained checkpoints when the causal language model is - used in combination with the [`EncoderDecoderModel`] framework. - """ - - def __init__(self, config): - super().__init__(config) - self.decoder = OPTDecoder(config) - - def forward(self, *args, **kwargs): - return self.decoder(*args, **kwargs) - - class OPTForCausalLM(OPTPretrainedModel): def __init__(self, config): config = copy.deepcopy(config) config.is_decoder = True config.is_encoder_decoder = False super().__init__(config) - self.model = OPTDecoderWrapper(config) + self.model = OPTModel(config) if config.output_projection: self.lm_head = self.model.decoder.output_projection @@ -1001,7 +947,7 @@ def set_decoder(self, decoder): def get_decoder(self): return self.model.decoder - @replace_return_docstrings(output_type=CausalLMOutputWithCrossAttentions, config_class=_CONFIG_FOR_DOC) + @replace_return_docstrings(output_type=CausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC) def forward( self, input_ids: torch.LongTensor = None, @@ -1015,7 +961,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, CausalLMOutputWithCrossAttentions]: + ) -> Union[Tuple, CausalLMOutputWithPast]: r""" Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): @@ -1128,13 +1074,12 @@ def forward( output = (logits,) + outputs[1:] return (loss,) + output if loss is not None else output - return CausalLMOutputWithCrossAttentions( + return CausalLMOutputWithPast( loss=loss, logits=logits, past_key_values=outputs.past_key_values, hidden_states=outputs.hidden_states, attentions=outputs.attentions, - cross_attentions=outputs.cross_attentions, ) def prepare_inputs_for_generation(self, input_ids, past=None, attention_mask=None, use_cache=None, **kwargs): diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index c23b853ad71dd..cc93f1848225a 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -27,14 +27,12 @@ from ...generation.test_generation_utils import GenerationTesterMixin from ...test_configuration_common import ConfigTester -from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor +from ...test_modeling_common import ModelTesterMixin, ids_tensor if is_torch_available(): import torch - from transformers import GPT2Tokenizer, OPTForCausalLM, OPTModel - from transformers.models.opt.modeling_opt import OPTDecoder, shift_tokens_right def prepare_opt_inputs_dict( @@ -45,7 +43,6 @@ def prepare_opt_inputs_dict( decoder_attention_mask=None, head_mask=None, decoder_head_mask=None, - cross_attn_head_mask=None, ): if attention_mask is None: attention_mask = input_ids.ne(config.pad_token_id) @@ -53,16 +50,10 @@ def prepare_opt_inputs_dict( decoder_attention_mask = decoder_input_ids.ne(config.pad_token_id) if decoder_head_mask is None: decoder_head_mask = torch.ones(config.num_hidden_layers, config.num_attention_heads, device=torch_device) - if cross_attn_head_mask is None: - cross_attn_head_mask = torch.ones(config.num_hidden_layers, config.num_attention_heads, device=torch_device) return { "input_ids": input_ids, - # "decoder_input_ids": decoder_input_ids, "attention_mask": attention_mask, - # "decoder_attention_mask": attention_mask, "head_mask": head_mask, - # "decoder_head_mask": decoder_head_mask, - "cross_attn_head_mask": cross_attn_head_mask, } @@ -158,7 +149,7 @@ def create_and_check_decoder_model_past_large_inputs(self, config, inputs_dict): # first forward pass outputs = model(input_ids, attention_mask=attention_mask, head_mask=head_mask, use_cache=True) - output, past_key_values = outputs.to_tuple() # TODO, TOO MANY VALUES TO UNPACK HERE + output, past_key_values = outputs.to_tuple() # create hypothetical multiple next token and extent to next_input_ids next_tokens = ids_tensor((self.batch_size, 3), config.vocab_size) @@ -262,15 +253,6 @@ def test_generate_beam_search(self): ) self.assertEqual(generated_ids.shape, (input_ids.shape[0], max_length)) - def test_shift_tokens_right(self): - input_ids = torch.tensor([[71, 82, 18, 33, 2, 1, 1], [68, 34, 26, 58, 30, 82, 2]], dtype=torch.long) - shifted = shift_tokens_right(input_ids, 1, 2) - n_pad_before = input_ids.eq(1).float().sum() - n_pad_after = shifted.eq(1).float().sum() - self.assertEqual(shifted.shape, input_ids.shape) - self.assertEqual(n_pad_after, n_pad_before - 1) - self.assertTrue(torch.eq(shifted[:, 0], 2).all()) - @slow def test_tokenization(self): tokenizer = GPT2Tokenizer.from_pretrained("facebook/opt-large") @@ -373,15 +355,15 @@ def test_inputs_embeds(self): with torch.no_grad(): model(**inputs)[0] - # def test_generate_fp16(self): - # config, input_dict = self.model_tester.prepare_config_and_inputs() - # input_ids = input_dict["input_ids"] - # attention_mask = input_ids.ne(1).to(torch_device) - # model = OPTForConditionalGeneration(config).eval().to(torch_device) - # if torch_device == "cuda": - # model.half() - # model.generate(input_ids, attention_mask=attention_mask) - # model.generate(num_beams=4, do_sample=True, early_stopping=False, num_return_sequences=3) + def test_generate_fp16(self): + config, input_dict = self.model_tester.prepare_config_and_inputs() + input_ids = input_dict["input_ids"] + attention_mask = input_ids.ne(1).to(torch_device) + model = OPTForCausalLM(config).eval().to(torch_device) + if torch_device == "cuda": + model.half() + model.generate(input_ids, attention_mask=attention_mask) + model.generate(num_beams=4, do_sample=True, early_stopping=False, num_return_sequences=3) def assert_tensors_close(a, b, atol=1e-12, prefix=""): @@ -407,70 +389,6 @@ def _long_tensor(tok_lst): return torch.tensor(tok_lst, dtype=torch.long, device=torch_device) -@require_torch -@slow -class FastIntegrationTests(unittest.TestCase): - """These tests are useful for debugging since they operate on a model with 1 encoder layer and 1 decoder layer.""" - - @cached_property - def tok(self): - return GPT2Tokenizer.from_pretrained("facebook/opt-large") - - @cached_property - def xsum_1_1_model(self): - return OPTForCausalLM.from_pretrained("sshleifer/distilopt-xsum-1-1") - - def test_xsum_1_1_generation(self): - hf = self.xsum_1_1_model - tok = self.tok - ARTICLE = 'The Palestinian Authority officially became the 123rd member of the International Criminal Court on Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian territories. The formal accession was marked with a ceremony at The Hague, in the Netherlands, where the court is based. The Palestinians signed the ICC\'s founding Rome Statute in January, when they also accepted its jurisdiction over alleged crimes committed "in the occupied Palestinian territory, including East Jerusalem, since June 13, 2014." Later that month, the ICC opened a preliminary examination into the situation in Palestinian territories, paving the way for possible war crimes investigations against Israelis. As members of the court, Palestinians may be subject to counter-charges as well. Israel and the United States, neither of which is an ICC member, opposed the Palestinians\' efforts to join the body. But Palestinian Foreign Minister Riad al-Malki, speaking at Wednesday\'s ceremony, said it was a move toward greater justice. "As Palestine formally becomes a State Party to the Rome Statute today, the world is also a step closer to ending a long era of impunity and injustice," he said, according to an ICC news release. "Indeed, today brings us closer to our shared goals of justice and peace." Judge Kuniko Ozaki, a vice president of the ICC, said acceding to the treaty was just the first step for the Palestinians. "As the Rome Statute today enters into force for the State of Palestine, Palestine acquires all the rights as well as responsibilities that come with being a State Party to the Statute. These are substantive commitments, which cannot be taken lightly," she said. Rights group Human Rights Watch welcomed the development. "Governments seeking to penalize Palestine for joining the ICC should immediately end their pressure, and countries that support universal acceptance of the court\'s treaty should speak out to welcome its membership," said Balkees Jarrah, international justice counsel for the group. "What\'s objectionable is the attempts to undermine international justice, not Palestine\'s decision to join a treaty to which over 100 countries around the world are members." In January, when the preliminary ICC examination was opened, Israeli Prime Minister Benjamin Netanyahu described it as an outrage, saying the court was overstepping its boundaries. The United States also said it "strongly" disagreed with the court\'s decision. "As we have said repeatedly, we do not believe that Palestine is a state and therefore we do not believe that it is eligible to join the ICC," the State Department said in a statement. It urged the warring sides to resolve their differences through direct negotiations. "We will continue to oppose actions against Israel at the ICC as counterproductive to the cause of peace," it said. But the ICC begs to differ with the definition of a state for its purposes and refers to the territories as "Palestine." While a preliminary examination is not a formal investigation, it allows the court to review evidence and determine whether to investigate suspects on both sides. Prosecutor Fatou Bensouda said her office would "conduct its analysis in full independence and impartiality." The war between Israel and Hamas militants in Gaza last summer left more than 2,000 people dead. The inquiry will include alleged war crimes committed since June. The International Criminal Court was set up in 2002 to prosecute genocide, crimes against humanity and war crimes.' - EXPECTED = " The International Criminal Court (ICC) has announced that it has been announced by the International Criminal court." - - dct = tok(ARTICLE, return_tensors="pt") - generated_ids = hf.generate(**dct, num_beams=4) - result = tok.batch_decode(generated_ids, skip_special_tokens=True)[0] - assert EXPECTED == result - - def test_xsum_1_1_batch_generation(self): - # test batch - - batch = self.tok( - [ - 'The Palestinian Authority officially became the 123rd member of the International Criminal Court on Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian territories. The formal accession was marked with a ceremony at The Hague, in the Netherlands, where the court is based. The Palestinians signed the ICC\'s founding Rome Statute in January, when they also accepted its jurisdiction over alleged crimes committed "in the occupied Palestinian territory, including East Jerusalem, since June 13, 2014." Later that month, the ICC opened a preliminary examination into the situation in Palestinian territories, paving the way for possible war crimes investigations against Israelis. As members of the court, Palestinians may be subject to counter-charges as well. Israel and the United States, neither of which is an ICC member, opposed the Palestinians\' efforts to join the body. But Palestinian Foreign Minister Riad al-Malki, speaking at Wednesday\'s ceremony, said it was a move toward greater justice. "As Palestine formally becomes a State Party to the Rome Statute today, the world is also a step closer to ending a long era of impunity and injustice," he said, according to an ICC news release. "Indeed, today brings us closer to our shared goals of justice and peace." Judge Kuniko Ozaki, a vice president of the ICC, said acceding to the treaty was just the first step for the Palestinians. "As the Rome Statute today enters into force for the State of Palestine, Palestine acquires all the rights as well as responsibilities that come with being a State Party to the Statute. These are substantive commitments, which cannot be taken lightly," she said. Rights group Human Rights Watch welcomed the development. "Governments seeking to penalize Palestine for joining the ICC should immediately end their pressure, and countries that support universal acceptance of the court\'s treaty should speak out to welcome its membership," said Balkees Jarrah, international justice counsel for the group. "What\'s objectionable is the attempts to undermine international justice, not Palestine\'s decision to join a treaty to which over 100 countries around the world are members." In January, when the preliminary ICC examination was opened, Israeli Prime Minister Benjamin Netanyahu described it as an outrage, saying the court was overstepping its boundaries. The United States also said it "strongly" disagreed with the court\'s decision. "As we have said repeatedly, we do not believe that Palestine is a state and therefore we do not believe that it is eligible to join the ICC," the State Department said in a statement. It urged the warring sides to resolve their differences through direct negotiations. "We will continue to oppose actions against Israel at the ICC as counterproductive to the cause of peace," it said. But the ICC begs to differ with the definition of a state for its purposes and refers to the territories as "Palestine." While a preliminary examination is not a formal investigation, it allows the court to review evidence and determine whether to investigate suspects on both sides. Prosecutor Fatou Bensouda said her office would "conduct its analysis in full independence and impartiality." The war between Israel and Hamas militants in Gaza last summer left more than 2,000 people dead. The inquiry will include alleged war crimes committed since June. The International Criminal Court was set up in 2002 to prosecute genocide, crimes against humanity and war crimes.', - 'The French prosecutor leading an investigation into the crash of Germanwings Flight 9525 insisted Wednesday that he was not aware of any video footage from on board the plane. Marseille prosecutor Brice Robin told CNN that "so far no videos were used in the crash investigation." He added, "A person who has such a video needs to immediately give it to the investigators." Robin\'s comments follow claims by two magazines, German daily Bild and French Paris Match, of a cell phone video showing the harrowing final seconds from on board Germanwings Flight 9525 as it crashed into the French Alps. All 150 on board were killed. Paris Match and Bild reported that the video was recovered from a phone at the wreckage site. The two publications described the supposed video, but did not post it on their websites. The publications said that they watched the video, which was found by a source close to the investigation. "One can hear cries of \'My God\' in several languages," Paris Match reported. "Metallic banging can also be heard more than three times, perhaps of the pilot trying to open the cockpit door with a heavy object. Towards the end, after a heavy shake, stronger than the others, the screaming intensifies. Then nothing." "It is a very disturbing scene," said Julian Reichelt, editor-in-chief of Bild online. An official with France\'s accident investigation agency, the BEA, said the agency is not aware of any such video. Lt. Col. Jean-Marc Menichini, a French Gendarmerie spokesman in charge of communications on rescue efforts around the Germanwings crash site, told CNN that the reports were "completely wrong" and "unwarranted." Cell phones have been collected at the site, he said, but that they "hadn\'t been exploited yet." Menichini said he believed the cell phones would need to be sent to the Criminal Research Institute in Rosny sous-Bois, near Paris, in order to be analyzed by specialized technicians working hand-in-hand with investigators. But none of the cell phones found so far have been sent to the institute, Menichini said. Asked whether staff involved in the search could have leaked a memory card to the media, Menichini answered with a categorical "no." Reichelt told "Erin Burnett: Outfront" that he had watched the video and stood by the report, saying Bild and Paris Match are "very confident" that the clip is real. He noted that investigators only revealed they\'d recovered cell phones from the crash site after Bild and Paris Match published their reports. "That is something we did not know before. ... Overall we can say many things of the investigation weren\'t revealed by the investigation at the beginning," he said. What was mental state of Germanwings co-pilot? German airline Lufthansa confirmed Tuesday that co-pilot Andreas Lubitz had battled depression years before he took the controls of Germanwings Flight 9525, which he\'s accused of deliberately crashing last week in the French Alps. Lubitz told his Lufthansa flight training school in 2009 that he had a "previous episode of severe depression," the airline said Tuesday. Email correspondence between Lubitz and the school discovered in an internal investigation, Lufthansa said, included medical documents he submitted in connection with resuming his flight training. The announcement indicates that Lufthansa, the parent company of Germanwings, knew of Lubitz\'s battle with depression, allowed him to continue training and ultimately put him in the cockpit. Lufthansa, whose CEO Carsten Spohr previously said Lubitz was 100% fit to fly, described its statement Tuesday as a "swift and seamless clarification" and said it was sharing the information and documents -- including training and medical records -- with public prosecutors. Spohr traveled to the crash site Wednesday, where recovery teams have been working for the past week to recover human remains and plane debris scattered across a steep mountainside. He saw the crisis center set up in Seyne-les-Alpes, laid a wreath in the village of Le Vernet, closer to the crash site, where grieving families have left flowers at a simple stone memorial. Menichini told CNN late Tuesday that no visible human remains were left at the site but recovery teams would keep searching. French President Francois Hollande, speaking Tuesday, said that it should be possible to identify all the victims using DNA analysis by the end of the week, sooner than authorities had previously suggested. In the meantime, the recovery of the victims\' personal belongings will start Wednesday, Menichini said. Among those personal belongings could be more cell phones belonging to the 144 passengers and six crew on board. Check out the latest from our correspondents . The details about Lubitz\'s correspondence with the flight school during his training were among several developments as investigators continued to delve into what caused the crash and Lubitz\'s possible motive for downing the jet. A Lufthansa spokesperson told CNN on Tuesday that Lubitz had a valid medical certificate, had passed all his examinations and "held all the licenses required." Earlier, a spokesman for the prosecutor\'s office in Dusseldorf, Christoph Kumpa, said medical records reveal Lubitz suffered from suicidal tendencies at some point before his aviation career and underwent psychotherapy before he got his pilot\'s license. Kumpa emphasized there\'s no evidence suggesting Lubitz was suicidal or acting aggressively before the crash. Investigators are looking into whether Lubitz feared his medical condition would cause him to lose his pilot\'s license, a European government official briefed on the investigation told CNN on Tuesday. While flying was "a big part of his life," the source said, it\'s only one theory being considered. Another source, a law enforcement official briefed on the investigation, also told CNN that authorities believe the primary motive for Lubitz to bring down the plane was that he feared he would not be allowed to fly because of his medical problems. Lubitz\'s girlfriend told investigators he had seen an eye doctor and a neuropsychologist, both of whom deemed him unfit to work recently and concluded he had psychological issues, the European government official said. But no matter what details emerge about his previous mental health struggles, there\'s more to the story, said Brian Russell, a forensic psychologist. "Psychology can explain why somebody would turn rage inward on themselves about the fact that maybe they weren\'t going to keep doing their job and they\'re upset about that and so they\'re suicidal," he said. "But there is no mental illness that explains why somebody then feels entitled to also take that rage and turn it outward on 149 other people who had nothing to do with the person\'s problems." Germanwings crash compensation: What we know . Who was the captain of Germanwings Flight 9525? CNN\'s Margot Haddad reported from Marseille and Pamela Brown from Dusseldorf, while Laura Smith-Spark wrote from London. CNN\'s Frederik Pleitgen, Pamela Boykoff, Antonia Mortensen, Sandrine Amiel and Anna-Maja Rappard contributed to this report.', - ], - return_tensors="pt", - padding="longest", - truncation=True, - ) - generated_ids = self.xsum_1_1_model.generate(**batch, num_beams=4) - result = self.tok.batch_decode(generated_ids, skip_special_tokens=True) - assert ( - result[0] - == " The International Criminal Court (ICC) has announced that it has been announced by the International Criminal court." - ) - assert ( - result[1] - == " An investigation into the crash that killed at least 10 people in the French capital has been released by the French police investigating the crash." - ) - - def test_encoder_equiv(self): - # test batch - - batch = self.tok( - [ - 'The Palestinian Authority officially became the 123rd member of the International Criminal Court on Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian territories. The formal accession was marked with a ceremony at The Hague, in the Netherlands, where the court is based. The Palestinians signed the ICC\'s founding Rome Statute in January, when they also accepted its jurisdiction over alleged crimes committed "in the occupied Palestinian territory, including East Jerusalem, since June 13, 2014." Later that month, the ICC opened a preliminary examination into the situation in Palestinian territories, paving the way for possible war crimes investigations against Israelis. As members of the court, Palestinians may be subject to counter-charges as well. Israel and the United States, neither of which is an ICC member, opposed the Palestinians\' efforts to join the body. But Palestinian Foreign Minister Riad al-Malki, speaking at Wednesday\'s ceremony, said it was a move toward greater justice. "As Palestine formally becomes a State Party to the Rome Statute today, the world is also a step closer to ending a long era of impunity and injustice," he said, according to an ICC news release. "Indeed, today brings us closer to our shared goals of justice and peace." Judge Kuniko Ozaki, a vice president of the ICC, said acceding to the treaty was just the first step for the Palestinians. "As the Rome Statute today enters into force for the State of Palestine, Palestine acquires all the rights as well as responsibilities that come with being a State Party to the Statute. These are substantive commitments, which cannot be taken lightly," she said. Rights group Human Rights Watch welcomed the development. "Governments seeking to penalize Palestine for joining the ICC should immediately end their pressure, and countries that support universal acceptance of the court\'s treaty should speak out to welcome its membership," said Balkees Jarrah, international justice counsel for the group. "What\'s objectionable is the attempts to undermine international justice, not Palestine\'s decision to join a treaty to which over 100 countries around the world are members." In January, when the preliminary ICC examination was opened, Israeli Prime Minister Benjamin Netanyahu described it as an outrage, saying the court was overstepping its boundaries. The United States also said it "strongly" disagreed with the court\'s decision. "As we have said repeatedly, we do not believe that Palestine is a state and therefore we do not believe that it is eligible to join the ICC," the State Department said in a statement. It urged the warring sides to resolve their differences through direct negotiations. "We will continue to oppose actions against Israel at the ICC as counterproductive to the cause of peace," it said. But the ICC begs to differ with the definition of a state for its purposes and refers to the territories as "Palestine." While a preliminary examination is not a formal investigation, it allows the court to review evidence and determine whether to investigate suspects on both sides. Prosecutor Fatou Bensouda said her office would "conduct its analysis in full independence and impartiality." The war between Israel and Hamas militants in Gaza last summer left more than 2,000 people dead. The inquiry will include alleged war crimes committed since June. The International Criminal Court was set up in 2002 to prosecute genocide, crimes against humanity and war crimes.', - 'The French prosecutor leading an investigation into the crash of Germanwings Flight 9525 insisted Wednesday that he was not aware of any video footage from on board the plane. Marseille prosecutor Brice Robin told CNN that "so far no videos were used in the crash investigation." He added, "A person who has such a video needs to immediately give it to the investigators." Robin\'s comments follow claims by two magazines, German daily Bild and French Paris Match, of a cell phone video showing the harrowing final seconds from on board Germanwings Flight 9525 as it crashed into the French Alps. All 150 on board were killed. Paris Match and Bild reported that the video was recovered from a phone at the wreckage site. The two publications described the supposed video, but did not post it on their websites. The publications said that they watched the video, which was found by a source close to the investigation. "One can hear cries of \'My God\' in several languages," Paris Match reported. "Metallic banging can also be heard more than three times, perhaps of the pilot trying to open the cockpit door with a heavy object. Towards the end, after a heavy shake, stronger than the others, the screaming intensifies. Then nothing." "It is a very disturbing scene," said Julian Reichelt, editor-in-chief of Bild online. An official with France\'s accident investigation agency, the BEA, said the agency is not aware of any such video. Lt. Col. Jean-Marc Menichini, a French Gendarmerie spokesman in charge of communications on rescue efforts around the Germanwings crash site, told CNN that the reports were "completely wrong" and "unwarranted." Cell phones have been collected at the site, he said, but that they "hadn\'t been exploited yet." Menichini said he believed the cell phones would need to be sent to the Criminal Research Institute in Rosny sous-Bois, near Paris, in order to be analyzed by specialized technicians working hand-in-hand with investigators. But none of the cell phones found so far have been sent to the institute, Menichini said. Asked whether staff involved in the search could have leaked a memory card to the media, Menichini answered with a categorical "no." Reichelt told "Erin Burnett: Outfront" that he had watched the video and stood by the report, saying Bild and Paris Match are "very confident" that the clip is real. He noted that investigators only revealed they\'d recovered cell phones from the crash site after Bild and Paris Match published their reports. "That is something we did not know before. ... Overall we can say many things of the investigation weren\'t revealed by the investigation at the beginning," he said. What was mental state of Germanwings co-pilot? German airline Lufthansa confirmed Tuesday that co-pilot Andreas Lubitz had battled depression years before he took the controls of Germanwings Flight 9525, which he\'s accused of deliberately crashing last week in the French Alps. Lubitz told his Lufthansa flight training school in 2009 that he had a "previous episode of severe depression," the airline said Tuesday. Email correspondence between Lubitz and the school discovered in an internal investigation, Lufthansa said, included medical documents he submitted in connection with resuming his flight training. The announcement indicates that Lufthansa, the parent company of Germanwings, knew of Lubitz\'s battle with depression, allowed him to continue training and ultimately put him in the cockpit. Lufthansa, whose CEO Carsten Spohr previously said Lubitz was 100% fit to fly, described its statement Tuesday as a "swift and seamless clarification" and said it was sharing the information and documents -- including training and medical records -- with public prosecutors. Spohr traveled to the crash site Wednesday, where recovery teams have been working for the past week to recover human remains and plane debris scattered across a steep mountainside. He saw the crisis center set up in Seyne-les-Alpes, laid a wreath in the village of Le Vernet, closer to the crash site, where grieving families have left flowers at a simple stone memorial. Menichini told CNN late Tuesday that no visible human remains were left at the site but recovery teams would keep searching. French President Francois Hollande, speaking Tuesday, said that it should be possible to identify all the victims using DNA analysis by the end of the week, sooner than authorities had previously suggested. In the meantime, the recovery of the victims\' personal belongings will start Wednesday, Menichini said. Among those personal belongings could be more cell phones belonging to the 144 passengers and six crew on board. Check out the latest from our correspondents . The details about Lubitz\'s correspondence with the flight school during his training were among several developments as investigators continued to delve into what caused the crash and Lubitz\'s possible motive for downing the jet. A Lufthansa spokesperson told CNN on Tuesday that Lubitz had a valid medical certificate, had passed all his examinations and "held all the licenses required." Earlier, a spokesman for the prosecutor\'s office in Dusseldorf, Christoph Kumpa, said medical records reveal Lubitz suffered from suicidal tendencies at some point before his aviation career and underwent psychotherapy before he got his pilot\'s license. Kumpa emphasized there\'s no evidence suggesting Lubitz was suicidal or acting aggressively before the crash. Investigators are looking into whether Lubitz feared his medical condition would cause him to lose his pilot\'s license, a European government official briefed on the investigation told CNN on Tuesday. While flying was "a big part of his life," the source said, it\'s only one theory being considered. Another source, a law enforcement official briefed on the investigation, also told CNN that authorities believe the primary motive for Lubitz to bring down the plane was that he feared he would not be allowed to fly because of his medical problems. Lubitz\'s girlfriend told investigators he had seen an eye doctor and a neuropsychologist, both of whom deemed him unfit to work recently and concluded he had psychological issues, the European government official said. But no matter what details emerge about his previous mental health struggles, there\'s more to the story, said Brian Russell, a forensic psychologist. "Psychology can explain why somebody would turn rage inward on themselves about the fact that maybe they weren\'t going to keep doing their job and they\'re upset about that and so they\'re suicidal," he said. "But there is no mental illness that explains why somebody then feels entitled to also take that rage and turn it outward on 149 other people who had nothing to do with the person\'s problems." Germanwings crash compensation: What we know . Who was the captain of Germanwings Flight 9525? CNN\'s Margot Haddad reported from Marseille and Pamela Brown from Dusseldorf, while Laura Smith-Spark wrote from London. CNN\'s Frederik Pleitgen, Pamela Boykoff, Antonia Mortensen, Sandrine Amiel and Anna-Maja Rappard contributed to this report.', - ], - return_tensors="pt", - padding="longest", - truncation=True, - ) - features = self.xsum_1_1_model.get_encoder()(**batch).last_hidden_state - expected = [[-0.0828, -0.0251, -0.0674], [0.1277, 0.3311, -0.0255], [0.2613, -0.0840, -0.2763]] - assert_tensors_close(features[0, :3, :3], torch.tensor(expected), atol=1e-3) - - @require_torch @require_sentencepiece @require_tokenizers @@ -493,300 +411,10 @@ def test_inference_no_head(self): ) self.assertTrue(torch.allclose(output[:, :3, :3], expected_slice, atol=1e-3)) - # @slow - # def test_base_mask_filling(self): - # pbase = pipeline(task="fill-mask", model="") - # src_text = [" I went to the ."] - # results = [x["token_str"] for x in pbase(src_text)] - # assert " bathroom" in results - - # @slow - # def test_large_mask_filling(self): - # plarge = pipeline(task="fill-mask", model="facebook/opt-large") - # src_text = [" I went to the ."] - # results = [x["token_str"] for x in plarge(src_text)] - # expected_results = [" bathroom", " gym", " wrong", " movies", " hospital"] - # self.assertListEqual(results, expected_results) - - # @slow - # def test_mnli_inference(self): - # example_b = [0, 31414, 232, 328, 740, 1140, 69, 46078, 1588, 2, 1] - # input_ids = _long_tensor([[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2], example_b]) - - # model = AutoModelForSequenceClassification.from_pretrained("facebook/opt-large-mnli").to( - # torch_device - # ) # eval called in from_pre - # attention_mask = input_ids.ne(model.config.pad_token_id) - # # Test that model hasn't changed - # with torch.no_grad(): - # outputs = model(input_ids=input_ids, attention_mask=attention_mask) - - # batched_logits = outputs.logits - # expected_shape = torch.Size((2, 3)) - # self.assertEqual(batched_logits.shape, expected_shape) - # expected_slice = torch.tensor([[0.1907, 1.4342, -1.0289]], device=torch_device) - # logits_arr = batched_logits[0].detach() - - # # Test that padding does not change results - # input_ids_no_pad = _long_tensor([example_b[:-1]]) - # attention_mask_no_pad = input_ids_no_pad.ne(model.config.pad_token_id) - - # with torch.no_grad(): - # logits2 = model(input_ids=input_ids_no_pad, attention_mask=attention_mask_no_pad).logits.squeeze() - # assert_tensors_close(batched_logits[1], logits2, atol=1e-3) - # assert_tensors_close(expected_slice, logits_arr, atol=1e-3) - - # def test_xsum_config_generation_params(self): - # config = OPTConfig.from_pretrained("facebook/opt-large-xsum") - # expected_params = dict(num_beams=6, do_sample=False, early_stopping=True, length_penalty=1.0) - # config_params = {k: getattr(config, k, "MISSING") for k, v in expected_params.items()} - # self.assertDictEqual(expected_params, config_params) - - -@require_torch -class OPTStandaloneDecoderModelTester: - def __init__( - self, - parent, - vocab_size=99, - batch_size=13, - d_model=16, - decoder_seq_length=7, - is_training=True, - is_decoder=True, - use_attention_mask=True, - use_cache=False, - use_labels=True, - decoder_start_token_id=2, - decoder_ffn_dim=32, - decoder_layers=24, - encoder_attention_heads=4, - decoder_attention_heads=4, - max_position_embeddings=30, - is_encoder_decoder=False, - embed_dim=16, - pad_token_id=0, - bos_token_id=1, - eos_token_id=2, - scope=None, - ): - self.parent = parent - self.batch_size = batch_size - self.decoder_seq_length = decoder_seq_length - # For common tests - self.seq_length = self.decoder_seq_length - self.is_training = is_training - self.use_attention_mask = use_attention_mask - self.use_labels = use_labels - - self.vocab_size = vocab_size - self.d_model = d_model - self.hidden_size = d_model - self.num_hidden_layers = decoder_layers - self.decoder_layers = decoder_layers - self.decoder_ffn_dim = decoder_ffn_dim - - self.num_attention_heads = decoder_attention_heads - self.eos_token_id = eos_token_id - self.bos_token_id = bos_token_id - self.pad_token_id = pad_token_id - self.decoder_start_token_id = decoder_start_token_id - self.use_cache = use_cache - self.max_position_embeddings = max_position_embeddings - self.is_encoder_decoder = is_encoder_decoder - self.embed_dim = embed_dim - self.scope = None - self.decoder_key_length = decoder_seq_length - self.base_model_out_len = 2 - self.decoder_attention_idx = 1 - - def prepare_config_and_inputs(self): - input_ids = ids_tensor([self.batch_size, self.decoder_seq_length], self.vocab_size) - - attention_mask = None - if self.use_attention_mask: - attention_mask = ids_tensor([self.batch_size, self.decoder_seq_length], vocab_size=2) - - lm_labels = None - if self.use_labels: - lm_labels = ids_tensor([self.batch_size, self.decoder_seq_length], self.vocab_size) - - config = OPTConfig( - vocab_size=self.vocab_size, - d_model=self.d_model, - num_hidden_layers=self.num_hidden_layers, - ffn_dim=self.decoder_ffn_dim, - num_attention_heads=self.num_attention_heads, - eos_token_id=self.eos_token_id, - bos_token_id=self.bos_token_id, - use_cache=self.use_cache, - pad_token_id=self.pad_token_id, - decoder_start_token_id=self.decoder_start_token_id, - max_position_embeddings=self.max_position_embeddings, - is_encoder_decoder=self.is_encoder_decoder, - embed_dim=self.embed_dim, - ) - - return ( - config, - input_ids, - attention_mask, - lm_labels, - ) - - def prepare_config_and_inputs_for_decoder(self): - ( - config, - input_ids, - attention_mask, - lm_labels, - ) = self.prepare_config_and_inputs() - - encoder_hidden_states = floats_tensor([self.batch_size, self.decoder_seq_length, self.hidden_size]) - encoder_attention_mask = ids_tensor([self.batch_size, self.decoder_seq_length], vocab_size=2) - - return ( - config, - input_ids, - attention_mask, - encoder_hidden_states, - encoder_attention_mask, - lm_labels, - ) - - def create_and_check_decoder_model_past( - self, - config, - input_ids, - attention_mask, - lm_labels, - ): - config.use_cache = True - model = OPTDecoder(config=config).to(torch_device).eval() - # first forward pass - outputs = model(input_ids, use_cache=True) - outputs_use_cache_conf = model(input_ids) - outputs_no_past = model(input_ids, use_cache=False) - - self.parent.assertTrue(len(outputs) == len(outputs_use_cache_conf)) - self.parent.assertTrue(len(outputs) == len(outputs_no_past) + 1) - - past_key_values = outputs["past_key_values"] - - # create hypothetical next token and extent to next_input_ids - next_tokens = ids_tensor((self.batch_size, 1), config.vocab_size) - - # append to next input_ids and - next_input_ids = torch.cat([input_ids, next_tokens], dim=-1) - - output_from_no_past = model(next_input_ids)["last_hidden_state"] - output_from_past = model(next_tokens, past_key_values=past_key_values)["last_hidden_state"] - - # select random slice - random_slice_idx = ids_tensor((1,), output_from_past.shape[-1]).item() - output_from_no_past_slice = output_from_no_past[:, next_input_ids.shape[-1] - 1, random_slice_idx].detach() - output_from_past_slice = output_from_past[:, 0, random_slice_idx].detach() - - # test that outputs are equal for slice - assert torch.allclose(output_from_past_slice, output_from_no_past_slice, atol=1e-3) - - def create_and_check_decoder_model_attention_mask_past( - self, - config, - input_ids, - attention_mask, - lm_labels, - ): - model = OPTDecoder(config=config).to(torch_device).eval() - - # create attention mask - attn_mask = torch.ones(input_ids.shape, dtype=torch.long, device=torch_device) - - half_seq_length = input_ids.shape[-1] // 2 - attn_mask[:, half_seq_length:] = 0 - - # first forward pass - past_key_values = model(input_ids, attention_mask=attn_mask, use_cache=True)["past_key_values"] - - # create hypothetical next token and extent to next_input_ids - next_tokens = ids_tensor((self.batch_size, 1), config.vocab_size) - - # change a random masked slice from input_ids - random_seq_idx_to_change = ids_tensor((1,), half_seq_length).item() + 1 - random_other_next_tokens = ids_tensor((self.batch_size, 1), config.vocab_size).squeeze(-1) - input_ids[:, -random_seq_idx_to_change] = random_other_next_tokens - - # append to next input_ids and attn_mask - next_input_ids = torch.cat([input_ids, next_tokens], dim=-1) - attn_mask = torch.cat( - [attn_mask, torch.ones((attn_mask.shape[0], 1), dtype=torch.long, device=torch_device)], - dim=1, - ) - - # get two different outputs - output_from_no_past = model(next_input_ids, attention_mask=attn_mask)["last_hidden_state"] - output_from_past = model(next_tokens, attention_mask=attn_mask, past_key_values=past_key_values)[ - "last_hidden_state" - ] - - # select random slice - random_slice_idx = ids_tensor((1,), output_from_past.shape[-1]).item() - output_from_no_past_slice = output_from_no_past[:, next_input_ids.shape[-1] - 1, random_slice_idx].detach() - output_from_past_slice = output_from_past[:, 0, random_slice_idx].detach() - - # test that outputs are equal for slice - assert torch.allclose(output_from_past_slice, output_from_no_past_slice, atol=1e-3) - - def prepare_config_and_inputs_for_common(self): - config_and_inputs = self.prepare_config_and_inputs() - ( - config, - input_ids, - attention_mask, - lm_labels, - ) = config_and_inputs - - inputs_dict = { - "input_ids": input_ids, - "attention_mask": attention_mask, - } - return config, inputs_dict - - -@require_torch -class OPTStandaloneDecoderModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase): - all_model_classes = (OPTDecoder, OPTForCausalLM) if is_torch_available() else () - all_generative_model_classes = (OPTForCausalLM,) if is_torch_available() else () - test_pruning = False - is_encoder_decoder = False - - def setUp( - self, - ): - self.model_tester = OPTStandaloneDecoderModelTester(self, is_training=False) - self.config_tester = ConfigTester(self, config_class=OPTConfig) - - def test_config(self): - self.config_tester.run_common_tests() - - def test_decoder_model_past(self): - config_and_inputs = self.model_tester.prepare_config_and_inputs() - self.model_tester.create_and_check_decoder_model_past(*config_and_inputs) - - def test_decoder_model_attn_mask_past(self): - config_and_inputs = self.model_tester.prepare_config_and_inputs() - self.model_tester.create_and_check_decoder_model_attention_mask_past(*config_and_inputs) - - def test_retain_grad_hidden_states_attentions(self): - # decoder cannot keep gradients - return - - -# TODO solve the @torch.no_grad() issue - @require_tokenizers @require_torch +@slow class OPTEmbeddingsTest(unittest.TestCase): def setUp(self): super().setUp() @@ -829,6 +457,7 @@ def test_logits(self): @require_tokenizers +@slow class OPTGenerationTest(unittest.TestCase): def setUp(self): super().setUp() From 5491431e5c3969cfabbb081bb665681d835fc39d Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Tue, 10 May 2022 15:54:32 +0200 Subject: [PATCH 073/145] remove linear head hack --- src/transformers/models/opt/modeling_opt.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 8b43c73a42875..a700e35d48fe6 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. """ PyTorch OPT model.""" -import copy import math import random from typing import Dict, List, Optional, Tuple, Union @@ -598,12 +597,6 @@ def __init__(self, config: OPTConfig): ) self.layer_norm = nn.LayerNorm(config.d_model) if config.decoder_layernorm else None - - if config.output_projection: - self.output_projection = nn.Linear(config.embed_dim, config.vocab_size, bias=False) - else: - self.output_projection = None - self.layers = nn.ModuleList([OPTDecoderLayer(config) for _ in range(config.num_hidden_layers)]) self.gradient_checkpointing = False @@ -914,17 +907,11 @@ def forward( class OPTForCausalLM(OPTPretrainedModel): def __init__(self, config): - config = copy.deepcopy(config) - config.is_decoder = True - config.is_encoder_decoder = False super().__init__(config) self.model = OPTModel(config) - if config.output_projection: - self.lm_head = self.model.decoder.output_projection - else: - self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False) - self.lm_head.weight = nn.Parameter(self.model.decoder.embed_tokens.weight.t()) + # the lm_head weight is automatically tied to the embed tokens weight + self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False) # Initialize weights and apply final processing self.post_init() From 521822f1665d8d60ea1f0e85f523260cdf679397 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Tue, 10 May 2022 16:00:55 +0200 Subject: [PATCH 074/145] save intermediate --- src/transformers/models/opt/modeling_opt.py | 231 +++++++++++++++----- 1 file changed, 178 insertions(+), 53 deletions(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index a700e35d48fe6..fcc0013a77ff3 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -135,7 +135,7 @@ def forward( ) -class OPTAttention(nn.Module): +class OLD_OPTAttention(nn.Module): """Multi-headed attention from 'Attention Is All You Need' paper""" def __init__( @@ -212,15 +212,11 @@ def forward( key_states = self._shape(self.k_proj(hidden_states), -1, bsz) value_states = self._shape(self.v_proj(hidden_states), -1, bsz) - if self.is_decoder: - # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. - # Further calls to cross_attention layer can then reuse all cross-attention - # key/value_states (first "if" case) - # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of - # all previous decoder key/value_states. Further calls to uni-directional self-attention - # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) - # if encoder bi-directional self-attention `past_key_value` is always `None` - past_key_value = (key_states, value_states) + # uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of + # all previous decoder key/value_states. Further calls to uni-directional self-attention + # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) + # if encoder bi-directional self-attention `past_key_value` is always `None` + past_key_value = (key_states, value_states) proj_shape = (bsz * self.num_heads, -1, self.head_dim) query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) @@ -256,22 +252,173 @@ def forward( else: raise RuntimeError(f"attn_mask's dimension {attention_mask.dim()} is not supported") - if key_padding_mask is not None: - assert key_padding_mask.shape == ( - bsz, - src_len, - ), f"expecting key_padding_mask shape of {(bsz, src_len)}, but got {key_padding_mask.shape}" - key_padding_mask = ( - key_padding_mask.view(bsz, 1, 1, src_len) - .expand(-1, self.num_heads, -1, -1) - .reshape(bsz * self.num_heads, 1, src_len) + if attention_mask is not None: + attn_weights = torch.baddbmm(attention_mask, query_states, key_states.transpose(-2, -1)) + else: + attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + + if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): + raise ValueError( + f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is {attn_weights.size()}" ) - if attention_mask is None: - attention_mask = key_padding_mask - elif attention_mask.dtype == torch.bool: - attention_mask = attention_mask.logical_or(key_padding_mask) + + attn_weights = nn.functional.softmax(attn_weights, dim=-1) + + if layer_head_mask is not None: + if layer_head_mask.size() != (self.num_heads,): + raise ValueError( + f"Head mask for a single layer should be of size {(self.num_heads,)}, but is {layer_head_mask.size()}" + ) + attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + if output_attentions: + # this operation is a bit awkward, but it's required to + # make sure that attn_weights keeps its gradient. + # In order to do so, attn_weights have to be reshaped + # twice and have to be reused in the following + attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) + else: + attn_weights_reshaped = None + + attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + + attn_output = torch.bmm(attn_probs, value_states) + + if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is {attn_output.size()}" + ) + + attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim) + attn_output = attn_output.transpose(1, 2) + + # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be + # partitioned aross GPUs when using tensor-parallelism. + attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim) + + attn_output = self.out_proj(attn_output) + + return attn_output, attn_weights_reshaped, past_key_value + + +class OPTAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + embed_dim: int, + num_heads: int, + dropout: float = 0.1, + is_decoder: bool = True, + bias: bool = True, + add_zero_attn=False, + ): + super().__init__() + self.embed_dim = embed_dim + self.num_heads = num_heads + self.dropout = dropout + self.head_dim = embed_dim // num_heads + self.add_zero_attn = add_zero_attn + + if (self.head_dim * num_heads) != self.embed_dim: + raise ValueError( + f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}" + f" and `num_heads`: {num_heads})." + ) + self.scaling = self.head_dim**-0.5 + self.is_decoder = is_decoder + + self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias) + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + key_value_states: Optional[torch.Tensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + attention_mask: Optional[torch.Tensor] = None, + layer_head_mask: Optional[torch.Tensor] = None, + key_padding_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + """Input shape: Batch x Time x Channel""" + + # if key_value_states are provided this layer is used as a cross-attention layer + # for the decoder + # hidden_states = hidden_states.permute(1, 0, 2) + + is_cross_attention = key_value_states is not None + + bsz, tgt_len, _ = hidden_states.size() + + # get query proj + query_states = self.q_proj(hidden_states) * self.scaling + # get key, value proj + if is_cross_attention and past_key_value is not None: + # reuse k,v, cross_attentions + key_states = past_key_value[0] + value_states = past_key_value[1] + elif is_cross_attention: + # cross_attentions + key_states = self._shape(self.k_proj(key_value_states), -1, bsz) + value_states = self._shape(self.v_proj(key_value_states), -1, bsz) + elif past_key_value is not None: + # reuse k, v, self_attention + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + key_states = torch.cat([past_key_value[0], key_states], dim=2) + value_states = torch.cat([past_key_value[1], value_states], dim=2) + else: + # self_attention + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + + # uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of + # all previous decoder key/value_states. Further calls to uni-directional self-attention + # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) + # if encoder bi-directional self-attention `past_key_value` is always `None` + past_key_value = (key_states, value_states) + + proj_shape = (bsz * self.num_heads, -1, self.head_dim) + query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) + key_states = key_states.view(*proj_shape) + value_states = value_states.view(*proj_shape) + + src_len = key_states.size(1) + + if attention_mask is not None: + if attention_mask.dtype == torch.uint8: + # warnings.warn("Byte tensor for attn_mask in nn.MultiheadAttention is deprecated. Use bool tensor instead.") + attention_mask = attention_mask.to(torch.bool) + else: + assert ( + attention_mask.is_floating_point() or attention_mask.dtype == torch.bool + ), f"Only float, byte, and bool types are supported for attn_mask, not {attention_mask.dtype}" + # ensure attn_mask's dim is 3 + if attention_mask.dim() == 2: + correct_2d_size = (tgt_len, src_len) + if attention_mask.shape != correct_2d_size: + raise RuntimeError( + f"The shape of the 2D attn_mask is {attention_mask.shape}, but should be {correct_2d_size}." + ) + attention_mask = attention_mask.unsqueeze(0) + elif attention_mask.dim() == 3: + correct_3d_size = (bsz * self.num_heads, tgt_len, src_len) + if attention_mask.shape != correct_3d_size: + raise RuntimeError( + f"The shape of the 3D attn_mask is {attention_mask.shape}, but should be {correct_3d_size}." + ) + elif attention_mask.dim() == 4: + attention_mask = attention_mask[0, 0, :, :] # consider only one mask since they should be all the same else: - attention_mask = attention_mask.masked_fill(key_padding_mask, float("-inf")) + raise RuntimeError(f"attn_mask's dimension {attention_mask.dim()} is not supported") if attention_mask is not None: attn_weights = torch.baddbmm(attention_mask, query_states, key_states.transpose(-2, -1)) @@ -363,8 +510,6 @@ def forward( `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size `(encoder_attention_heads,)`. - cross_attn_layer_head_mask (`torch.FloatTensor`): mask for cross-attention heads in a given layer of - size `(num_attention_heads,)`. past_key_value (`Tuple(torch.FloatTensor)`): cached past key and value projection states output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under @@ -396,8 +541,6 @@ def forward( hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) hidden_states = residual + hidden_states hidden_states = self.self_attn_layer_norm(hidden_states) - # Cross-Attention Block - cross_attn_weights = None # Fully Connected hidden_states_shape = hidden_states.shape @@ -405,18 +548,17 @@ def forward( residual = hidden_states hidden_states = self.fc1(hidden_states) hidden_states = self.activation_fn(hidden_states) - # hidden_states = nn.functional.dropout(hidden_states, p=self.activation_dropout, training=self.training) + hidden_states = self.fc2(hidden_states) hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) hidden_states = (residual + hidden_states).view(hidden_states_shape) - # hidden_states = residual + hidden_states hidden_states = self.final_layer_norm(hidden_states) outputs = (hidden_states,) if output_attentions: - outputs += (self_attn_weights, cross_attn_weights) + outputs += (self_attn_weights,) if use_cache: outputs += (present_key_value,) @@ -633,7 +775,6 @@ def forward( attention_mask: Optional[torch.Tensor] = None, head_mask: Optional[torch.Tensor] = None, self_attn_padding_mask: Optional[torch.Tensor] = None, - cross_attn_head_mask: Optional[torch.Tensor] = None, past_key_values: Optional[List[torch.FloatTensor]] = None, inputs_embeds: Optional[torch.FloatTensor] = None, use_cache: Optional[bool] = None, @@ -664,13 +805,6 @@ def forward( - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. - cross_attn_head_mask (`torch.Tensor` of shape `(num_hidden_layers, num_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules in the decoder to avoid performing - cross-attention on hidden heads. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of @@ -741,11 +875,10 @@ def forward( # decoder layers all_hidden_states = () if output_hidden_states else None all_self_attns = () if output_attentions else None - all_cross_attentions = () if (output_attentions is not None) else None next_decoder_cache = () if use_cache else None - # check if head_mask/cross_attn_head_mask has a correct number of layers specified if desired - for attn_mask, mask_name in zip([head_mask, cross_attn_head_mask], ["head_mask", "cross_attn_head_mask"]): + # check if head_mask has a correct number of layers specified if desired + for attn_mask, mask_name in zip([head_mask], ["head_mask"]): if attn_mask is not None: if attn_mask.size()[0] != (len(self.layers)): raise ValueError( @@ -782,7 +915,6 @@ def custom_forward(*inputs): hidden_states, attention_mask, head_mask[idx] if head_mask is not None else None, - cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None, None, ) else: @@ -820,7 +952,7 @@ def custom_forward(*inputs): if not return_dict: return tuple( v - for v in [hidden_states, next_cache, all_hidden_states, all_self_attns, all_cross_attentions] + for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None ) return BaseModelOutputWithPast( @@ -940,7 +1072,6 @@ def forward( input_ids: torch.LongTensor = None, attention_mask: Optional[torch.Tensor] = None, head_mask: Optional[torch.Tensor] = None, - cross_attn_head_mask: Optional[torch.Tensor] = None, past_key_values: Optional[List[torch.FloatTensor]] = None, inputs_embeds: Optional[torch.FloatTensor] = None, labels: Optional[torch.LongTensor] = None, @@ -972,12 +1103,6 @@ def forward( - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. - cross_attn_head_mask (`torch.Tensor` of shape `(num_hidden_layers, num_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of @@ -1016,8 +1141,9 @@ def forward( ```python >>> from transformers import OPTTokenizer, OPTForCausalLM + # this needs fixing >>> tokenizer = OPTTokenizer.from_pretrained("") - >>> model = OPTForCausalLM.from_pretrained("", add_cross_attention=False) + >>> model = OPTForCausalLM.from_pretrained("") >>> assert model.config.is_decoder, f"{model.__class__} has to be configured as a decoder." >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") >>> outputs = model(**inputs) @@ -1039,7 +1165,6 @@ def forward( input_ids=input_ids, attention_mask=attention_mask, head_mask=head_mask, - cross_attn_head_mask=cross_attn_head_mask, past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache, From 61e80232406e7d76a007a4de089c4d1f5f8e3385 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Tue, 10 May 2022 16:02:01 +0200 Subject: [PATCH 075/145] save correct attention --- src/transformers/models/opt/modeling_opt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index fcc0013a77ff3..19885cadd09f6 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -303,6 +303,7 @@ def forward( return attn_output, attn_weights_reshaped, past_key_value +# Copied from transformers.models.bart.modeling_bart.BartAttention with Bart->OPT class OPTAttention(nn.Module): """Multi-headed attention from 'Attention Is All You Need' paper""" From 7b27a912b18344e0920b42338bd7c48431768e31 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Tue, 10 May 2022 16:14:56 +0200 Subject: [PATCH 076/145] add copied from from bart --- src/transformers/models/opt/modeling_opt.py | 263 ++------------------ 1 file changed, 21 insertions(+), 242 deletions(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 19885cadd09f6..bccf96259c1e5 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -135,24 +135,23 @@ def forward( ) -class OLD_OPTAttention(nn.Module): +# Copied from transformers.models.bart.modeling_bart.BartAttention with Bart->OPT +class OPTAttention(nn.Module): """Multi-headed attention from 'Attention Is All You Need' paper""" def __init__( self, embed_dim: int, num_heads: int, - dropout: float = 0.1, - is_decoder: bool = True, + dropout: float = 0.0, + is_decoder: bool = False, bias: bool = True, - add_zero_attn=False, ): super().__init__() self.embed_dim = embed_dim self.num_heads = num_heads self.dropout = dropout self.head_dim = embed_dim // num_heads - self.add_zero_attn = add_zero_attn if (self.head_dim * num_heads) != self.embed_dim: raise ValueError( @@ -177,15 +176,12 @@ def forward( past_key_value: Optional[Tuple[torch.Tensor]] = None, attention_mask: Optional[torch.Tensor] = None, layer_head_mask: Optional[torch.Tensor] = None, - key_padding_mask: Optional[torch.Tensor] = None, output_attentions: bool = False, ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: """Input shape: Batch x Time x Channel""" # if key_value_states are provided this layer is used as a cross-attention layer # for the decoder - # hidden_states = hidden_states.permute(1, 0, 2) - is_cross_attention = key_value_states is not None bsz, tgt_len, _ = hidden_states.size() @@ -212,11 +208,15 @@ def forward( key_states = self._shape(self.k_proj(hidden_states), -1, bsz) value_states = self._shape(self.v_proj(hidden_states), -1, bsz) - # uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of - # all previous decoder key/value_states. Further calls to uni-directional self-attention - # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) - # if encoder bi-directional self-attention `past_key_value` is always `None` - past_key_value = (key_states, value_states) + if self.is_decoder: + # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. + # Further calls to cross_attention layer can then reuse all cross-attention + # key/value_states (first "if" case) + # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of + # all previous decoder key/value_states. Further calls to uni-directional self-attention + # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) + # if encoder bi-directional self-attention `past_key_value` is always `None` + past_key_value = (key_states, value_states) proj_shape = (bsz * self.num_heads, -1, self.head_dim) query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) @@ -224,213 +224,21 @@ def forward( value_states = value_states.view(*proj_shape) src_len = key_states.size(1) - - if attention_mask is not None: - if attention_mask.dtype == torch.uint8: - # warnings.warn("Byte tensor for attn_mask in nn.MultiheadAttention is deprecated. Use bool tensor instead.") - attention_mask = attention_mask.to(torch.bool) - else: - assert ( - attention_mask.is_floating_point() or attention_mask.dtype == torch.bool - ), f"Only float, byte, and bool types are supported for attn_mask, not {attention_mask.dtype}" - # ensure attn_mask's dim is 3 - if attention_mask.dim() == 2: - correct_2d_size = (tgt_len, src_len) - if attention_mask.shape != correct_2d_size: - raise RuntimeError( - f"The shape of the 2D attn_mask is {attention_mask.shape}, but should be {correct_2d_size}." - ) - attention_mask = attention_mask.unsqueeze(0) - elif attention_mask.dim() == 3: - correct_3d_size = (bsz * self.num_heads, tgt_len, src_len) - if attention_mask.shape != correct_3d_size: - raise RuntimeError( - f"The shape of the 3D attn_mask is {attention_mask.shape}, but should be {correct_3d_size}." - ) - elif attention_mask.dim() == 4: - attention_mask = attention_mask[0, 0, :, :] # consider only one mask since they should be all the same - else: - raise RuntimeError(f"attn_mask's dimension {attention_mask.dim()} is not supported") - - if attention_mask is not None: - attn_weights = torch.baddbmm(attention_mask, query_states, key_states.transpose(-2, -1)) - else: - attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): raise ValueError( f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is {attn_weights.size()}" ) - attn_weights = nn.functional.softmax(attn_weights, dim=-1) - - if layer_head_mask is not None: - if layer_head_mask.size() != (self.num_heads,): + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, tgt_len, src_len): raise ValueError( - f"Head mask for a single layer should be of size {(self.num_heads,)}, but is {layer_head_mask.size()}" + f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" ) - attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) - if output_attentions: - # this operation is a bit awkward, but it's required to - # make sure that attn_weights keeps its gradient. - # In order to do so, attn_weights have to be reshaped - # twice and have to be reused in the following - attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) - attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) - else: - attn_weights_reshaped = None - - attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) - - attn_output = torch.bmm(attn_probs, value_states) - - if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): - raise ValueError( - f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is {attn_output.size()}" - ) - - attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim) - attn_output = attn_output.transpose(1, 2) - - # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be - # partitioned aross GPUs when using tensor-parallelism. - attn_output = attn_output.reshape(bsz, tgt_len, self.embed_dim) - - attn_output = self.out_proj(attn_output) - - return attn_output, attn_weights_reshaped, past_key_value - - -# Copied from transformers.models.bart.modeling_bart.BartAttention with Bart->OPT -class OPTAttention(nn.Module): - """Multi-headed attention from 'Attention Is All You Need' paper""" - - def __init__( - self, - embed_dim: int, - num_heads: int, - dropout: float = 0.1, - is_decoder: bool = True, - bias: bool = True, - add_zero_attn=False, - ): - super().__init__() - self.embed_dim = embed_dim - self.num_heads = num_heads - self.dropout = dropout - self.head_dim = embed_dim // num_heads - self.add_zero_attn = add_zero_attn - - if (self.head_dim * num_heads) != self.embed_dim: - raise ValueError( - f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}" - f" and `num_heads`: {num_heads})." - ) - self.scaling = self.head_dim**-0.5 - self.is_decoder = is_decoder - - self.k_proj = nn.Linear(embed_dim, embed_dim, bias=bias) - self.v_proj = nn.Linear(embed_dim, embed_dim, bias=bias) - self.q_proj = nn.Linear(embed_dim, embed_dim, bias=bias) - self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias) - - def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): - return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() - - def forward( - self, - hidden_states: torch.Tensor, - key_value_states: Optional[torch.Tensor] = None, - past_key_value: Optional[Tuple[torch.Tensor]] = None, - attention_mask: Optional[torch.Tensor] = None, - layer_head_mask: Optional[torch.Tensor] = None, - key_padding_mask: Optional[torch.Tensor] = None, - output_attentions: bool = False, - ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: - """Input shape: Batch x Time x Channel""" - - # if key_value_states are provided this layer is used as a cross-attention layer - # for the decoder - # hidden_states = hidden_states.permute(1, 0, 2) - - is_cross_attention = key_value_states is not None - - bsz, tgt_len, _ = hidden_states.size() - - # get query proj - query_states = self.q_proj(hidden_states) * self.scaling - # get key, value proj - if is_cross_attention and past_key_value is not None: - # reuse k,v, cross_attentions - key_states = past_key_value[0] - value_states = past_key_value[1] - elif is_cross_attention: - # cross_attentions - key_states = self._shape(self.k_proj(key_value_states), -1, bsz) - value_states = self._shape(self.v_proj(key_value_states), -1, bsz) - elif past_key_value is not None: - # reuse k, v, self_attention - key_states = self._shape(self.k_proj(hidden_states), -1, bsz) - value_states = self._shape(self.v_proj(hidden_states), -1, bsz) - key_states = torch.cat([past_key_value[0], key_states], dim=2) - value_states = torch.cat([past_key_value[1], value_states], dim=2) - else: - # self_attention - key_states = self._shape(self.k_proj(hidden_states), -1, bsz) - value_states = self._shape(self.v_proj(hidden_states), -1, bsz) - - # uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of - # all previous decoder key/value_states. Further calls to uni-directional self-attention - # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) - # if encoder bi-directional self-attention `past_key_value` is always `None` - past_key_value = (key_states, value_states) - - proj_shape = (bsz * self.num_heads, -1, self.head_dim) - query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) - key_states = key_states.view(*proj_shape) - value_states = value_states.view(*proj_shape) - - src_len = key_states.size(1) - - if attention_mask is not None: - if attention_mask.dtype == torch.uint8: - # warnings.warn("Byte tensor for attn_mask in nn.MultiheadAttention is deprecated. Use bool tensor instead.") - attention_mask = attention_mask.to(torch.bool) - else: - assert ( - attention_mask.is_floating_point() or attention_mask.dtype == torch.bool - ), f"Only float, byte, and bool types are supported for attn_mask, not {attention_mask.dtype}" - # ensure attn_mask's dim is 3 - if attention_mask.dim() == 2: - correct_2d_size = (tgt_len, src_len) - if attention_mask.shape != correct_2d_size: - raise RuntimeError( - f"The shape of the 2D attn_mask is {attention_mask.shape}, but should be {correct_2d_size}." - ) - attention_mask = attention_mask.unsqueeze(0) - elif attention_mask.dim() == 3: - correct_3d_size = (bsz * self.num_heads, tgt_len, src_len) - if attention_mask.shape != correct_3d_size: - raise RuntimeError( - f"The shape of the 3D attn_mask is {attention_mask.shape}, but should be {correct_3d_size}." - ) - elif attention_mask.dim() == 4: - attention_mask = attention_mask[0, 0, :, :] # consider only one mask since they should be all the same - else: - raise RuntimeError(f"attn_mask's dimension {attention_mask.dim()} is not supported") - - if attention_mask is not None: - attn_weights = torch.baddbmm(attention_mask, query_states, key_states.transpose(-2, -1)) - else: - attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) - - if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): - raise ValueError( - f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is {attn_weights.size()}" - ) - attn_weights = nn.functional.softmax(attn_weights, dim=-1) if layer_head_mask is not None: @@ -501,7 +309,6 @@ def forward( layer_head_mask: Optional[torch.Tensor] = None, output_attentions: Optional[bool] = False, use_cache: Optional[bool] = False, - key_padding_mask: Optional[Tensor] = None, past_key_value: Optional[Tuple[torch.Tensor]] = None, ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]: """ @@ -520,24 +327,12 @@ def forward( residual = hidden_states # Self Attention - # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 - # self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None - # add present self-attn cache to positions 1,2 of present_key_value tuple - # hidden_states, self_attn_weights, present_key_value = self.self_attn( - # hidden_states=hidden_states, - # attention_mask=attention_mask, - # self_attn_padding_mask=self_attn_padding_mask, - # layer_head_mask=layer_head_mask, - # output_attentions=output_attentions, - # past_key_value=past_key_value, - # ) hidden_states, self_attn_weights, present_key_value = self.self_attn( hidden_states=hidden_states, past_key_value=past_key_value, attention_mask=attention_mask, layer_head_mask=layer_head_mask, output_attentions=output_attentions, - key_padding_mask=key_padding_mask, ) hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) hidden_states = residual + hidden_states @@ -609,17 +404,6 @@ def _set_gradient_checkpointing(self, module, value=False): if isinstance(module, (OPTDecoder)): module.gradient_checkpointing = value - @property - @add_start_docstrings_to_model_forward(OPT_START_DOCSTRING) - def dummy_inputs(self): - pad_token = self.config.pad_token_id - input_ids = torch.tensor([[0, 6, 10, 4, 2], [0, 8, 12, 2, pad_token]], device=self.device) - dummy_inputs = { - "attention_mask": input_ids.ne(pad_token), - "input_ids": input_ids, - } - return dummy_inputs - OPT_GENERATION_EXAMPLE = r""" Generation example: @@ -775,7 +559,6 @@ def forward( input_ids: torch.LongTensor = None, attention_mask: Optional[torch.Tensor] = None, head_mask: Optional[torch.Tensor] = None, - self_attn_padding_mask: Optional[torch.Tensor] = None, past_key_values: Optional[List[torch.FloatTensor]] = None, inputs_embeds: Optional[torch.FloatTensor] = None, use_cache: Optional[bool] = None, @@ -848,8 +631,6 @@ def forward( else: raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds") - # past_key_values_length - past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0 if inputs_embeds is None: @@ -859,9 +640,6 @@ def forward( attention_mask, input_shape, inputs_embeds, past_key_values_length ) - if self_attn_padding_mask is None and (self.cross_self_attention or input_ids.eq(self.padding_idx).any()): - self_attn_padding_mask = input_ids.eq(self.padding_idx) - # embed positions positions = self.embed_positions(input_ids) @@ -871,7 +649,6 @@ def forward( hidden_states = inputs_embeds + positions hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) - # hidden_states = hidden_states.permute(1, 0, 2) # decoder layers all_hidden_states = () if output_hidden_states else None @@ -927,7 +704,6 @@ def custom_forward(*inputs): past_key_value=past_key_value, output_attentions=output_attentions, use_cache=use_cache, - key_padding_mask=self_attn_padding_mask, ) if self.layer_norm: @@ -1039,8 +815,11 @@ def forward( class OPTForCausalLM(OPTPretrainedModel): + _keys_to_ignore_on_load_missing = [r"lm_head\.weight"] + def __init__(self, config): super().__init__(config) + self.model = OPTModel(config) # the lm_head weight is automatically tied to the embed tokens weight From 766145365e3f2800edec3e43282de6accde92816 Mon Sep 17 00:00:00 2001 From: Arthur <48595927+ArthurZucker@users.noreply.github.com> Date: Wed, 11 May 2022 08:13:55 +0200 Subject: [PATCH 077/145] Update src/transformers/models/opt/modeling_opt.py Co-authored-by: Patrick von Platen --- src/transformers/models/opt/modeling_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index bccf96259c1e5..13590214c7962 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -90,7 +90,7 @@ def make_positions(tensor, padding_idx: int, onnx_trace: bool = False): # balanced to both work with ONNX export and XLA. In particular XLA # prefers ints, cumsum defaults to output longs, and ONNX doesn't know # how to handle the dtype kwarg in cumsum. - mask = tensor.ne(padding_idx).int() + mask = attention_mask return (torch.cumsum(mask, dim=1).type_as(mask) * mask).long() + padding_idx From 25a40b1e48808a47362ba872908a877ae614c3e6 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Wed, 11 May 2022 08:42:42 +0200 Subject: [PATCH 078/145] fix part of the reviewss Co-authored-by: Patrick von Platen --- .../models/opt/configuration_opt.py | 223 +----------------- src/transformers/models/opt/modeling_opt.py | 36 ++- tests/models/opt/test_modeling_opt.py | 125 +--------- 3 files changed, 17 insertions(+), 367 deletions(-) diff --git a/src/transformers/models/opt/configuration_opt.py b/src/transformers/models/opt/configuration_opt.py index 79bfe93c7c72f..acaf6fcf520eb 100644 --- a/src/transformers/models/opt/configuration_opt.py +++ b/src/transformers/models/opt/configuration_opt.py @@ -105,7 +105,7 @@ def __init__( num_attention_heads=16, ffn_dim=4096, layerdrop=0.0, - activation_function="gelu", + activation_function="relu", d_model=1024, embed_dim=512, dropout=0.1, @@ -158,224 +158,3 @@ def __init__( f"Please make sure the config includes `forced_bos_token_id={self.bos_token_id}` in future versions. " "The config can simply be saved and uploaded again to be fixed." ) - - -class OPTOnnxConfig(OnnxSeq2SeqConfigWithPast): - @property - def inputs(self) -> Mapping[str, Mapping[int, str]]: - if self.task in ["default", "seq2seq-lm"]: - common_inputs = OrderedDict( - [ - ("input_ids", {0: "batch", 1: "encoder_sequence"}), - ("attention_mask", {0: "batch", 1: "encoder_sequence"}), - ] - ) - - if self.use_past: - common_inputs["decoder_input_ids"] = {0: "batch"} - common_inputs["decoder_attention_mask"] = {0: "batch", 1: "past_decoder_sequence + sequence"} - else: - common_inputs["decoder_input_ids"] = {0: "batch", 1: "decoder_sequence"} - common_inputs["decoder_attention_mask"] = {0: "batch", 1: "decoder_sequence"} - - if self.use_past: - self.fill_with_past_key_values_(common_inputs, direction="inputs") - elif self.task == "causal-lm": - # TODO: figure this case out. - common_inputs = OrderedDict( - [ - ("input_ids", {0: "batch", 1: "encoder_sequence"}), - ("attention_mask", {0: "batch", 1: "encoder_sequence"}), - ] - ) - if self.use_past: - num_encoder_layers, _ = self.num_hidden_layers - for i in range(num_encoder_layers): - common_inputs[f"past_key_values.{i}.key"] = {0: "batch", 2: "past_sequence + sequence"} - common_inputs[f"past_key_values.{i}.value"] = {0: "batch", 2: "past_sequence + sequence"} - else: - common_inputs = OrderedDict( - [ - ("input_ids", {0: "batch", 1: "encoder_sequence"}), - ("attention_mask", {0: "batch", 1: "encoder_sequence"}), - ("decoder_input_ids", {0: "batch", 1: "decoder_sequence"}), - ("decoder_attention_mask", {0: "batch", 1: "decoder_sequence"}), - ] - ) - - return common_inputs - - @property - def outputs(self) -> Mapping[str, Mapping[int, str]]: - if self.task in ["default", "seq2seq-lm"]: - common_outputs = super().outputs - else: - common_outputs = super(OnnxConfigWithPast, self).outputs - if self.use_past: - num_encoder_layers, _ = self.num_hidden_layers - for i in range(num_encoder_layers): - common_outputs[f"present.{i}.key"] = {0: "batch", 2: "past_sequence + sequence"} - common_outputs[f"present.{i}.value"] = {0: "batch", 2: "past_sequence + sequence"} - return common_outputs - - def _generate_dummy_inputs_for_default_and_seq2seq_lm( - self, - tokenizer: PreTrainedTokenizer, - batch_size: int = -1, - seq_length: int = -1, - is_pair: bool = False, - framework: Optional[TensorType] = None, - ) -> Mapping[str, Any]: - encoder_inputs = self._generate_dummy_inputs_for_sequence_classification_and_question_answering( - tokenizer, batch_size, seq_length, is_pair, framework - ) - - # Generate decoder inputs - decoder_seq_length = seq_length if not self.use_past else 1 - decoder_inputs = self._generate_dummy_inputs_for_sequence_classification_and_question_answering( - tokenizer, batch_size, decoder_seq_length, is_pair, framework - ) - decoder_inputs = {f"decoder_{name}": tensor for name, tensor in decoder_inputs.items()} - common_inputs = dict(**encoder_inputs, **decoder_inputs) - - if self.use_past: - if not is_torch_available(): - raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed.") - else: - import torch - batch, encoder_seq_length = common_inputs["input_ids"].shape - decoder_seq_length = common_inputs["decoder_input_ids"].shape[1] - num_encoder_attention_heads, num_num_attention_heads = self.num_attention_heads - encoder_shape = ( - batch, - num_encoder_attention_heads, - encoder_seq_length, - self._config.hidden_size // num_encoder_attention_heads, - ) - decoder_past_length = decoder_seq_length + 3 - decoder_shape = ( - batch, - num_num_attention_heads, - decoder_past_length, - self._config.hidden_size // num_num_attention_heads, - ) - - common_inputs["decoder_attention_mask"] = torch.cat( - [common_inputs["decoder_attention_mask"], torch.ones(batch, decoder_past_length)], dim=1 - ) - - common_inputs["past_key_values"] = [] - # If the number of encoder and decoder layers are present in the model configuration, both are considered - num_encoder_layers, num_hidden_layers = self.num_hidden_layers - min_num_hidden_layers = min(num_encoder_layers, num_hidden_layers) - max_num_hidden_layers = max(num_encoder_layers, num_hidden_layers) - min_num_hidden_layers - remaining_side_name = "encoder" if num_encoder_layers > num_hidden_layers else "decoder" - - for _ in range(min_num_hidden_layers): - common_inputs["past_key_values"].append( - ( - torch.zeros(decoder_shape), - torch.zeros(decoder_shape), - torch.zeros(encoder_shape), - torch.zeros(encoder_shape), - ) - ) - # TODO: test this. - shape = encoder_shape if remaining_side_name == "encoder" else decoder_shape - for _ in range(min_num_hidden_layers, max_num_hidden_layers): - common_inputs["past_key_values"].append((torch.zeros(shape), torch.zeros(shape))) - return common_inputs - - def _generate_dummy_inputs_for_causal_lm( - self, - tokenizer: PreTrainedTokenizer, - batch_size: int = -1, - seq_length: int = -1, - is_pair: bool = False, - framework: Optional[TensorType] = None, - ) -> Mapping[str, Any]: - common_inputs = self._generate_dummy_inputs_for_sequence_classification_and_question_answering( - tokenizer, batch_size, seq_length, is_pair, framework - ) - - if self.use_past: - if not is_torch_available(): - raise ValueError("Cannot generate dummy past_keys inputs without PyTorch installed.") - else: - import torch - batch, seqlen = common_inputs["input_ids"].shape - # Not using the same length for past_key_values - past_key_values_length = seqlen + 2 - num_encoder_layers, _ = self.num_hidden_layers - num_encoder_attention_heads, _ = self.num_attention_heads - past_shape = ( - batch, - num_encoder_attention_heads, - past_key_values_length, - self._config.hidden_size // num_encoder_attention_heads, - ) - - common_inputs["attention_mask"] = torch.cat( - [common_inputs["attention_mask"], torch.ones(batch, past_key_values_length)], dim=1 - ) - common_inputs["past_key_values"] = [ - (torch.zeros(past_shape), torch.zeros(past_shape)) for _ in range(num_encoder_layers) - ] - return common_inputs - - def _generate_dummy_inputs_for_sequence_classification_and_question_answering( - self, - tokenizer: PreTrainedTokenizer, - batch_size: int = -1, - seq_length: int = -1, - is_pair: bool = False, - framework: Optional[TensorType] = None, - ) -> Mapping[str, Any]: - # Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity. - # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX - batch_size = compute_effective_axis_dimension( - batch_size, fixed_dimension=OnnxConfig.default_fixed_batch, num_token_to_add=0 - ) - - # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX - token_to_add = tokenizer.num_special_tokens_to_add(is_pair) - seq_length = compute_effective_axis_dimension( - seq_length, fixed_dimension=OnnxConfig.default_fixed_sequence, num_token_to_add=token_to_add - ) - - # Generate dummy inputs according to compute batch and sequence - dummy_input = [" ".join([tokenizer.unk_token]) * seq_length] * batch_size - common_inputs = dict(tokenizer(dummy_input, return_tensors=framework)) - return common_inputs - - def generate_dummy_inputs( - self, - tokenizer: PreTrainedTokenizer, - batch_size: int = -1, - seq_length: int = -1, - is_pair: bool = False, - framework: Optional[TensorType] = None, - ) -> Mapping[str, Any]: - if self.task in ["default", "seq2seq-lm"]: - common_inputs = self._generate_dummy_inputs_for_default_and_seq2seq_lm( - tokenizer, batch_size=batch_size, seq_length=seq_length, is_pair=is_pair, framework=framework - ) - - elif self.task == "causal-lm": - common_inputs = self._generate_dummy_inputs_for_causal_lm( - tokenizer, batch_size=batch_size, seq_length=seq_length, is_pair=is_pair, framework=framework - ) - else: - common_inputs = self._generate_dummy_inputs_for_sequence_classification_and_question_answering( - tokenizer, batch_size=batch_size, seq_length=seq_length, is_pair=is_pair, framework=framework - ) - - return common_inputs - - def _flatten_past_key_values_(self, flattened_output, name, idx, t): - if self.task in ["default", "seq2seq-lm"]: - flattened_output = super()._flatten_past_key_values_(flattened_output, name, idx, t) - else: - flattened_output = super(OnnxSeq2SeqConfigWithPast, self)._flatten_past_key_values_( - flattened_output, name, idx, t - ) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 13590214c7962..59245a0538650 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -81,7 +81,7 @@ def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] return inverted_mask.masked_fill(inverted_mask.bool(), torch.finfo(dtype).min) -def make_positions(tensor, padding_idx: int, onnx_trace: bool = False): +def make_positions(attention_mask, padding_idx: int): """Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols are ignored. @@ -111,11 +111,10 @@ def __init__(self, num_embeddings: int, embedding_dim: int, padding_idx: int = 1 def forward( self, - input: Tensor, - incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]] = None, + attention_mask: Tensor, positions: Optional[Tensor] = None, ): - """Input is expected to be of size [bsz x seqlen].""" + """attention_masks is expected to be of size [bsz x seqlen].""" if not ((positions is None) or (self.padding_idx is None)): raise ValueError("If positions is pre-computed then padding_idx should not be set.") @@ -123,7 +122,7 @@ def forward( # padding. if positions is None: - positions = make_positions(input, self.padding_idx, onnx_trace=self.onnx_trace) + positions = make_positions(attention_mask, self.padding_idx) return F.embedding( positions, self.weight, @@ -292,8 +291,7 @@ def __init__(self, config: OPTConfig): is_decoder=True, ) self.dropout = config.dropout - self.activation_fn = ACT2FN["relu"] - # self.activation_fn = ACT2FN[config.activation_function] + self.activation_fn = ACT2FN[config.activation_function] self.activation_dropout = config.activation_dropout @@ -411,15 +409,15 @@ def _set_gradient_checkpointing(self, module, value=False): ```python >>> from transformers import GPT2Tokenizer, OPTForCausalLM - >>> model = OPTForCausalLM.from_pretrained("facebook/opt-large-cnn") - >>> tokenizer = GPT2Tokenizer.from_pretrained("facebook/opt-large-cnn") + >>> model = OPTForCausalLM.from_pretrained("ArthurZ/opt-350m") + >>> tokenizer = GPT2Tokenizer.from_pretrained("patrickvonplaten/opt_gpt2_tokenizer") >>> TEXTS_TO_GENERATE = "Hey, are you consciours? Can you talk to me?" "Hi there, my name is Barack" >>> inputs = tokenizer([TEXTS_TO_GENERATE], max_length=1024, return_tensors="pt") - >>> # Generate Summary - >>> summary_ids = model.generate(inputs["input_ids"], num_beams=2, min_length=0, max_length=20) - >>> tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] + >>> # Generate + >>> generate_ids = model.generate(inputs["input_ids"], num_beams=2, min_length=0, max_length=20) + >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] 'I'm not conscious.<\s>' ``` """ @@ -445,8 +443,7 @@ def _set_gradient_checkpointing(self, module, value=False): Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. - OPT uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` - is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_inputs`] and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the @@ -502,13 +499,12 @@ def __init__(self, config: OPTConfig): self.max_target_positions = config.max_position_embeddings self.share_input_output_embed = config.share_input_output_embed self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0 - self.cross_self_attention = False # TODO add it on the config self.vocab_size = config.vocab_size self.embed_tokens = nn.Embedding(config.vocab_size, config.embed_dim, self.padding_idx) if self.padding_idx is not None: - num_embeddings = config.max_position_embeddings + self.padding_idx + 1 + num_embeddings = config.max_position_embeddings + 2 self.embed_positions = OPTLearnedPositionalEmbedding( num_embeddings, config.d_model, @@ -641,7 +637,7 @@ def forward( ) # embed positions - positions = self.embed_positions(input_ids) + positions = self.embed_positions(attention_mask) if self.project_in_dim is not None: inputs_embeds = self.project_in_dim(inputs_embeds) @@ -791,8 +787,6 @@ def forward( use_cache = use_cache if use_cache is not None else self.config.use_cache return_dict = return_dict if return_dict is not None else self.config.use_return_dict - # If the user passed a tuple for encoder_outputs, we wrap it in a BaseModelOutputWithPastAndCrossAttentions when return_dict=True - # decoder outputs consists of (dec_features, past_key_value, dec_hidden, dec_attn) decoder_outputs = self.decoder( input_ids=input_ids, @@ -922,8 +916,8 @@ def forward( >>> from transformers import OPTTokenizer, OPTForCausalLM # this needs fixing - >>> tokenizer = OPTTokenizer.from_pretrained("") - >>> model = OPTForCausalLM.from_pretrained("") + >>> tokenizer = OPTTokenizer.from_pretrained("patrickvonplaten/opt_gpt2_tokenizer") + >>> model = OPTForCausalLM.from_pretrained("ArthurZ/opt-350m") >>> assert model.config.is_decoder, f"{model.__class__} has to be configured as a decoder." >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") >>> outputs = model(**inputs) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index cc93f1848225a..95ca7decc30b4 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -140,8 +140,7 @@ def prepare_config_and_inputs_for_common(self): def create_and_check_decoder_model_past_large_inputs(self, config, inputs_dict): model = OPTModel(config=config).to(torch_device).eval() - # previousdly OPTModel(config=config).get_decoder().to(torch_device).eval() - # tried OPTForCausalkML + input_ids = inputs_dict["input_ids"] attention_mask = inputs_dict["attention_mask"] head_mask = inputs_dict["head_mask"] @@ -175,127 +174,6 @@ def create_and_check_decoder_model_past_large_inputs(self, config, inputs_dict): self.parent.assertTrue(torch.allclose(output_from_past_slice, output_from_no_past_slice, atol=1e-3)) -@require_torch -class OPTHeadTests(unittest.TestCase): - vocab_size = 99 - - def _get_config_and_data(self): - input_ids = torch.tensor( - [ - [71, 82, 18, 33, 46, 91, 2], - [68, 34, 26, 58, 30, 82, 2], - [5, 97, 17, 39, 94, 40, 2], - [76, 83, 94, 25, 70, 78, 2], - [87, 59, 41, 35, 48, 66, 2], - [55, 13, 16, 58, 5, 2, 1], # note padding - [64, 27, 31, 51, 12, 75, 2], - [52, 64, 86, 17, 83, 39, 2], - [48, 61, 9, 24, 71, 82, 2], - [26, 1, 60, 48, 22, 13, 2], - [21, 5, 62, 28, 14, 76, 2], - [45, 98, 37, 86, 59, 48, 2], - [70, 70, 50, 9, 28, 0, 2], - ], - dtype=torch.long, - device=torch_device, - ) - - batch_size = input_ids.shape[0] - config = OPTConfig( - vocab_size=self.vocab_size, - d_model=24, - num_hidden_layers=2, - num_attention_heads=2, - ffn_dim=32, - max_position_embeddings=48, - eos_token_id=2, - pad_token_id=1, - bos_token_id=0, - embed_dim=24, - ) - return config, input_ids, batch_size - - @timeout_decorator.timeout(1) - def test_lm_forward(self): - config, input_ids, batch_size = self._get_config_and_data() - lm_labels = ids_tensor([batch_size, input_ids.shape[1]], self.vocab_size).to(torch_device) - lm_model = OPTForCausalLM(config) - lm_model.to(torch_device) - outputs = lm_model(input_ids=input_ids, labels=lm_labels) - expected_shape = (batch_size, input_ids.shape[1], config.vocab_size) - self.assertEqual(outputs["logits"].shape, expected_shape) - self.assertIsInstance(outputs["loss"].item(), float) - - def test_generate_beam_search(self): - input_ids = torch.tensor([[71, 82, 2], [68, 34, 2]], device=torch_device, dtype=torch.long) - config = OPTConfig( - vocab_size=self.vocab_size, - d_model=24, - num_hidden_layers=2, - num_attention_heads=2, - ffn_dim=32, - max_position_embeddings=48, - eos_token_id=2, - pad_token_id=1, - bos_token_id=0, - ) - lm_model = OPTForCausalLM(config).to(torch_device) - lm_model.eval() - - max_length = 5 - generated_ids = lm_model.generate( - input_ids.clone(), - do_sample=True, - num_return_sequences=1, - num_beams=2, - no_repeat_ngram_size=3, - max_length=max_length, - ) - self.assertEqual(generated_ids.shape, (input_ids.shape[0], max_length)) - - @slow - def test_tokenization(self): - tokenizer = GPT2Tokenizer.from_pretrained("facebook/opt-large") - examples = [" Hello world", " DomDramg"] # need leading spaces for equality - fairseq_results = [ - torch.tensor([0, 20920, 232, 2]), - torch.tensor([0, 11349, 495, 4040, 571, 2]), - ] - for ex, desired_result in zip(examples, fairseq_results): - opt_toks = tokenizer.encode(ex, return_tensors="pt").squeeze() - assert_tensors_close(desired_result.long(), opt_toks, prefix=ex) - - def test_generate_fp16(self): - config, input_ids, batch_size = self._get_config_and_data() - attention_mask = input_ids.ne(1).to(torch_device) - model = OPTForCausalLM(config).eval().to(torch_device) - if torch_device == "cuda": - model.half() - model.generate(input_ids, attention_mask=attention_mask) - model.generate(num_beams=4, do_sample=True, early_stopping=False, num_return_sequences=3) - - def test_dummy_inputs(self): - config, *_ = self._get_config_and_data() - model = OPTForCausalLM(config).eval().to(torch_device) - model(**model.dummy_inputs) - - def test_resize_tokens_embeddings_more(self): - config, input_ids, _ = self._get_config_and_data() - - def _get_embs(m): - return (m.get_input_embeddings().weight.data.clone(), m.get_output_embeddings().weight.data.clone()) - - model = OPTForCausalLM(config).eval().to(torch_device) - input, output = _get_embs(model) - self.assertTrue(torch.eq(input, output).all()) - new_vocab_size = 45 - model.resize_token_embeddings(new_vocab_size) - input_new, output_new = _get_embs(model) - self.assertEqual(input_new.shape, (new_vocab_size, config.d_model)) - self.assertEqual(output_new.shape, (new_vocab_size, config.d_model)) - self.assertTrue(torch.eq(input_new, output_new).all()) - - @require_torch class OPTModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCase): all_model_classes = (OPTModel, OPTForCausalLM) if is_torch_available() else () @@ -325,7 +203,6 @@ def test_decoder_model_past_with_large_inputs(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_decoder_model_past_large_inputs(*config_and_inputs) - # OPTForSequenceClassification does not support inputs_embeds def test_inputs_embeds(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() From f3b5e246dadaad79f5aadad146449df743bb754c Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Wed, 11 May 2022 11:10:52 +0200 Subject: [PATCH 079/145] same changes in naming / conversion --- .../models/opt/configuration_opt.py | 42 +++++++------------ ..._original_pytorch_checkpoint_to_pytorch.py | 42 +++++-------------- src/transformers/models/opt/modeling_opt.py | 39 +++++++++-------- 3 files changed, 48 insertions(+), 75 deletions(-) diff --git a/src/transformers/models/opt/configuration_opt.py b/src/transformers/models/opt/configuration_opt.py index acaf6fcf520eb..6f68e1026008c 100644 --- a/src/transformers/models/opt/configuration_opt.py +++ b/src/transformers/models/opt/configuration_opt.py @@ -1,5 +1,5 @@ # coding=utf-8 -# Copyright 2021 The Fairseq Authors and The HuggingFace Inc. team. All rights reserved. +# Copyright 2022 The Metaseq Authors and The HuggingFace Inc. team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,14 +14,9 @@ # limitations under the License. """ OPT model configuration""" import warnings -from collections import OrderedDict -from typing import Any, Mapping, Optional -from ... import PreTrainedTokenizer from ...configuration_utils import PretrainedConfig -from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeq2SeqConfigWithPast -from ...onnx.utils import compute_effective_axis_dimension -from ...utils import TensorType, is_torch_available, logging +from ...utils import logging logger = logging.get_logger(__name__) @@ -43,18 +38,18 @@ class OPTConfig(PretrainedConfig): Args: - vocab_size (`int`, *optional*, defaults to 50265): + vocab_size (`int`, *optional*, defaults to 50272): Vocabulary size of the OPT model. Defines the number of different tokens that can be represented by the `inputs_ids` passed when calling [`OPTModel`] or [`TFOPTModel`]. - d_model (`int`, *optional*, defaults to 1024): + d_model (`int`, *optional*, defaults to 768): Dimensionality of the layers and the pooler layer. num_hidden_layers (`int`, *optional*, defaults to 12): Number of decoder layers. - ffn_dim (`int`, *optional*, defaults to 4096): + ffn_dim (`int`, *optional*, defaults to 3072): Dimensionality of the "intermediate" (often named feed-forward) layer in decoder. - num_attention_heads (`int`, *optional*, defaults to 16): + num_attention_heads (`int`, *optional*, defaults to 12): Number of attention heads for each attention layer in the Transformer decoder. - activation_function (`str` or `function`, *optional*, defaults to `"gelu"`): + activation_function (`str` or `function`, *optional*, defaults to `"relu"`): The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, `"relu"`, `"silu"` and `"gelu_new"` are supported. dropout (`float`, *optional*, defaults to 0.1): @@ -63,7 +58,7 @@ class OPTConfig(PretrainedConfig): The dropout ratio for the attention probabilities. activation_dropout (`float`, *optional*, defaults to 0.0): The dropout ratio for activations inside the fully connected layer. - max_position_embeddings (`int`, *optional*, defaults to 1024): + max_position_embeddings (`int`, *optional*, defaults to 2048): The maximum sequence length that this model might ever be used with. Typically set this to something large just in case (e.g., 512 or 1024 or 2048). init_std (`float`, *optional*, defaults to 0.02): @@ -71,13 +66,8 @@ class OPTConfig(PretrainedConfig): layerdrop: (`float`, *optional*, defaults to 0.0): The LayerDrop probability. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556) for more details. - scale_embedding (`bool`, *optional*, defaults to `False`): - Scale embeddings by diving by sqrt(d_model). use_cache (`bool`, *optional*, defaults to `True`): Whether or not the model should return the last key/values attentions (not used by all models). - forced_eos_token_id (`int`, *optional*, defaults to 2): - The id of the token to force as the last generated token when `max_length` is reached. Usually set to - `eos_token_id`. Example: @@ -99,15 +89,15 @@ class OPTConfig(PretrainedConfig): def __init__( self, - vocab_size=50272, # TODO check the real value + vocab_size=50272, max_position_embeddings=2048, - num_hidden_layers=24, - num_attention_heads=16, - ffn_dim=4096, + num_hidden_layers=12, + num_attention_heads=12, + ffn_dim=3072, layerdrop=0.0, activation_function="relu", - d_model=1024, - embed_dim=512, + d_model=768, + word_embed_proj_dim=768, dropout=0.1, attention_dropout=0.0, activation_dropout=0.0, @@ -121,13 +111,13 @@ def __init__( decoder_start_token_id=2, forced_eos_token_id=2, output_projection=True, - decoder_layernorm=False, + decoder_layernorm=True, **kwargs ): self.vocab_size = vocab_size self.max_position_embeddings = max_position_embeddings self.num_attention_heads = num_attention_heads - self.embed_dim = embed_dim + self.word_embed_proj_dim = word_embed_proj_dim self.ffn_dim = ffn_dim self.share_input_output_embed = share_input_output_embed self.d_model = d_model diff --git a/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py index 82dc147dbf29a..cf98607b74a83 100644 --- a/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py @@ -1,5 +1,5 @@ # coding=utf-8 -# Copyright 2020 The HuggingFace Inc. team. +# Copyright 2022 The HuggingFace Inc. team. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,7 +18,6 @@ import argparse from pathlib import Path -# import fairseq import torch from torch import nn @@ -26,24 +25,9 @@ from transformers.utils import logging -FAIRSEQ_MODELS = ["opt.large", "opt.large.mnli", "opt.large.cnn", "opt_xsum/model.pt"] -# if version.parse(fairseq.__version__) < version.parse("0.9.0"): -# raise Exception("requires fairseq >= 0.9.0") - - logging.set_verbosity_info() logger = logging.get_logger(__name__) -SAMPLE_TEXT = " Hello world! cécé herlolip" - -mnli_rename_keys = [ - ("model.classification_heads.mnli.dense.weight", "classification_head.dense.weight"), - ("model.classification_heads.mnli.dense.bias", "classification_head.dense.bias"), - ("model.classification_heads.mnli.out_proj.weight", "classification_head.out_proj.weight"), - ("model.classification_heads.mnli.out_proj.bias", "classification_head.out_proj.bias"), -] - - def remove_ignore_keys_(state_dict): ignore_keys = [ "encoder.version", @@ -61,28 +45,24 @@ def rename_key(dct, old, new): dct[new] = val -def load_xsum_checkpoint(checkpoint_path): +def load_checkpoint(checkpoint_path): """Checkpoint path should end in model.pt""" sd = torch.load(checkpoint_path, map_location="cpu") if "model" in sd.keys(): sd = torch.load(checkpoint_path, map_location="cpu")["model"] - sd.pop("decoder.version") - return sd - -def make_linear_from_emb(emb): - vocab_size, emb_size = emb.weight.shape - lin_layer = nn.Linear(vocab_size, emb_size, bias=False) - lin_layer.weight.data = emb.weight.data - return lin_layer + # pop unnecessary weights + if "decoder.version" in sd: + sd.pop("decoder.version") + return sd @torch.no_grad() -def convert_opt_checkpoint(checkpoint_path, pytorch_dump_folder_path, hf_checkpoint_name=None): +def convert_opt_checkpoint(checkpoint_path, pytorch_dump_folder_path, config=None): """ Copy/paste/tweak model's weights to our BERT structure. """ - state_dict = load_xsum_checkpoint(checkpoint_path) + state_dict = load_checkpoint(checkpoint_path) config = OPTConfig() @@ -98,11 +78,11 @@ def convert_opt_checkpoint(checkpoint_path, pytorch_dump_folder_path, hf_checkpo parser = argparse.ArgumentParser() # Required parameters parser.add_argument( - "--fairseq_path", type=str, help="opt.large, opt.large.cnn or a path to a model.pt on local filesystem." + "--fairseq_path", type=str, help="path to fairseq checkpoint in correct format. You can find all checkpoints in the correct format here: https://huggingface.co/models?other=opt_metasq" ) parser.add_argument("--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model.") parser.add_argument( - "--hf_config", default=None, type=str, help="Which huggingface architecture to use: opt-large-xsum" + "--hf_config", default=None, type=str, help="Define HF config." ) args = parser.parse_args() - convert_opt_checkpoint(args.fairseq_path, args.pytorch_dump_folder_path, hf_checkpoint_name=args.hf_config) + convert_opt_checkpoint(args.fairseq_path, args.pytorch_dump_folder_path, config=args.hf_config) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 59245a0538650..83f703f622551 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -15,7 +15,7 @@ """ PyTorch OPT model.""" import math import random -from typing import Dict, List, Optional, Tuple, Union +from typing import List, Optional, Tuple, Union import torch import torch.nn.functional as F @@ -123,6 +123,7 @@ def forward( if positions is None: positions = make_positions(attention_mask, self.padding_idx) + return F.embedding( positions, self.weight, @@ -283,7 +284,6 @@ class OPTDecoderLayer(nn.Module): def __init__(self, config: OPTConfig): super().__init__() self.embed_dim = config.d_model - self.self_attn = OPTAttention( embed_dim=self.embed_dim, num_heads=config.num_attention_heads, @@ -415,7 +415,7 @@ def _set_gradient_checkpointing(self, module, value=False): >>> TEXTS_TO_GENERATE = "Hey, are you consciours? Can you talk to me?" "Hi there, my name is Barack" >>> inputs = tokenizer([TEXTS_TO_GENERATE], max_length=1024, return_tensors="pt") - >>> # Generate + >>> # Generate >>> generate_ids = model.generate(inputs["input_ids"], num_beams=2, min_length=0, max_length=20) >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] 'I'm not conscious.<\s>' @@ -501,7 +501,7 @@ def __init__(self, config: OPTConfig): self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0 self.vocab_size = config.vocab_size - self.embed_tokens = nn.Embedding(config.vocab_size, config.embed_dim, self.padding_idx) + self.embed_tokens = nn.Embedding(config.vocab_size, config.word_embed_proj_dim, self.padding_idx) if self.padding_idx is not None: num_embeddings = config.max_position_embeddings + 2 @@ -511,13 +511,15 @@ def __init__(self, config: OPTConfig): self.padding_idx, ) - self.project_out_dim = ( - nn.Linear(config.d_model, config.embed_dim, bias=False) if config.embed_dim != config.d_model else None - ) + if config.word_embed_proj_dim != config.d_model: + self.project_out = nn.Linear(config.d_model, config.word_embed_proj_dim, bias=False) + else: + self.project_out = None - self.project_in_dim = ( - nn.Linear(config.embed_dim, config.d_model, bias=False) if config.d_model != config.embed_dim else None - ) + if config.word_embed_proj_dim != config.d_model: + self.project_in = nn.Linear(config.word_embed_proj_dim, config.d_model, bias=False) + else: + self.project_in = None self.layer_norm = nn.LayerNorm(config.d_model) if config.decoder_layernorm else None self.layers = nn.ModuleList([OPTDecoderLayer(config) for _ in range(config.num_hidden_layers)]) @@ -639,8 +641,8 @@ def forward( # embed positions positions = self.embed_positions(attention_mask) - if self.project_in_dim is not None: - inputs_embeds = self.project_in_dim(inputs_embeds) + if self.project_in is not None: + inputs_embeds = self.project_in(inputs_embeds) hidden_states = inputs_embeds + positions @@ -702,10 +704,11 @@ def custom_forward(*inputs): use_cache=use_cache, ) - if self.layer_norm: - new_layer_output = (self.layer_norm(layer_outputs[0]),) - new_layer_output += (layer_outputs[1:],) - layer_outputs = new_layer_output + if self.layer_norm: + new_layer_output = (self.layer_norm(layer_outputs[0]),) + new_layer_output += (layer_outputs[1:],) + layer_outputs = new_layer_output + hidden_states = layer_outputs[0] if use_cache: @@ -714,8 +717,8 @@ def custom_forward(*inputs): if output_attentions: all_self_attns += (layer_outputs[1],) - if self.project_out_dim is not None: - hidden_states = self.project_out_dim(hidden_states) + if self.project_out is not None: + hidden_states = self.project_out(hidden_states) # add hidden states from the last decoder layer if output_hidden_states: From 0365e27e363dce8a9e78269a7bbe8ed9fc1ba5da Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Wed, 11 May 2022 11:14:03 +0200 Subject: [PATCH 080/145] correct mask --- src/transformers/models/opt/modeling_opt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 83f703f622551..f521d1e02a2f3 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -634,13 +634,13 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale + # embed positions + positions = self.embed_positions(attention_mask) + attention_mask = self._prepare_decoder_attention_mask( attention_mask, input_shape, inputs_embeds, past_key_values_length ) - # embed positions - positions = self.embed_positions(attention_mask) - if self.project_in is not None: inputs_embeds = self.project_in(inputs_embeds) From 929be2308a1783de1a27eb6f1b6dfde88e6f4586 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Wed, 11 May 2022 11:27:17 +0200 Subject: [PATCH 081/145] more fixes --- .../opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py | 4 ++-- src/transformers/models/opt/modeling_opt.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py index cf98607b74a83..c42aa7281aadc 100644 --- a/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py @@ -19,7 +19,6 @@ from pathlib import Path import torch -from torch import nn from transformers import OPTConfig, OPTModel from transformers.utils import logging @@ -28,6 +27,7 @@ logging.set_verbosity_info() logger = logging.get_logger(__name__) + def remove_ignore_keys_(state_dict): ignore_keys = [ "encoder.version", @@ -66,7 +66,7 @@ def convert_opt_checkpoint(checkpoint_path, pytorch_dump_folder_path, config=Non config = OPTConfig() - model = OPTModel(config).eval() + model = OPTModel(config).half().eval() model.load_state_dict(state_dict) # Check results diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index f521d1e02a2f3..eda19a47e2a43 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -635,6 +635,7 @@ def forward( inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale # embed positions + attention_mask = attention_mask if attention_mask is not None else torch.ones(inputs_embeds.shape[:2], dtype=torch.bool, device=inputs_embeds.device) positions = self.embed_positions(attention_mask) attention_mask = self._prepare_decoder_attention_mask( From f6b032b32942d0a100f395125e6592b9f92452eb Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Wed, 11 May 2022 11:41:15 +0200 Subject: [PATCH 082/145] delete FlaxOPT and TfOPT --- .../models/opt/modeling_flax_opt.py | 1318 ----------------- .../models/opt/modeling_tf_opt.py | 1215 --------------- src/transformers/utils/dummy_flax_objects.py | 28 - src/transformers/utils/dummy_tf_objects.py | 14 - tests/models/opt/test_modeling_flax_opt.py | 451 ------ tests/models/opt/test_modeling_tf_opt.py | 325 ---- 6 files changed, 3351 deletions(-) delete mode 100644 src/transformers/models/opt/modeling_flax_opt.py delete mode 100644 src/transformers/models/opt/modeling_tf_opt.py delete mode 100644 tests/models/opt/test_modeling_flax_opt.py delete mode 100644 tests/models/opt/test_modeling_tf_opt.py diff --git a/src/transformers/models/opt/modeling_flax_opt.py b/src/transformers/models/opt/modeling_flax_opt.py deleted file mode 100644 index 7f4f497e3678b..0000000000000 --- a/src/transformers/models/opt/modeling_flax_opt.py +++ /dev/null @@ -1,1318 +0,0 @@ -# coding=utf-8 -# Copyright 2021 The Fairseq Authors and The Google Flax Team Authors And The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" Flax OPT model.""" - -import math -import random -from functools import partial -from typing import Optional, Tuple - -import numpy as np - -import flax.linen as nn -import jax -import jax.numpy as jnp -from flax.core.frozen_dict import FrozenDict, freeze, unfreeze -from flax.linen import combine_masks, make_causal_mask -from flax.linen.attention import dot_product_attention_weights -from flax.traverse_util import flatten_dict, unflatten_dict -from jax import lax -from jax.random import PRNGKey - -from ...modeling_flax_outputs import ( - FlaxBaseModelOutput, - FlaxBaseModelOutputWithPastAndCrossAttentions, - FlaxCausalLMOutputWithCrossAttentions, - FlaxSeq2SeqModelOutput, -) -from ...modeling_flax_utils import ACT2FN, FlaxPreTrainedModel, append_call_sample_docstring -from ...utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings -from .configuration_opt import OPTConfig - - -logger = logging.get_logger(__name__) - -_CHECKPOINT_FOR_DOC = "" -_CONFIG_FOR_DOC = "OPTConfig" -_TOKENIZER_FOR_DOC = "GPT2Tokenizer" - - -OPT_START_DOCSTRING = r""" - This model inherits from [`FlaxPreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a Flax Linen - [flax.nn.Module](https://flax.readthedocs.io/en/latest/_autosummary/flax.nn.module.html) subclass. Use it as a - regular Flax Module and refer to the Flax documentation for all matter related to general usage and behavior. - - Finally, this model supports inherent JAX features such as: - - - [Just-In-Time (JIT) compilation](https://jax.readthedocs.io/en/latest/jax.html#just-in-time-compilation-jit) - - [Automatic Differentiation](https://jax.readthedocs.io/en/latest/jax.html#automatic-differentiation) - - [Vectorization](https://jax.readthedocs.io/en/latest/jax.html#vectorization-vmap) - - [Parallelization](https://jax.readthedocs.io/en/latest/jax.html#parallelization-pmap) - - Parameters: - config ([`OPTConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~FlaxPreTrainedModel.from_pretrained`] method to load the model weights. - dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`): - The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and - `jax.numpy.bfloat16` (on TPUs). - - This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If - specified all the computation will be performed with the given `dtype`. - - **Note that this only specifies the dtype of the computation and does not influence the dtype of model - parameters.** - - If you wish to change the dtype of the model parameters, see [`~FlaxPreTrainedModel.to_fp16`] and - [`~FlaxPreTrainedModel.to_bf16`]. -""" - -OPT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`jnp.ndarray` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - For translation and summarization training, `decoder_input_ids` should be provided. If no - `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right - for denoising pre-training following the paper. - decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - - If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the - paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. - position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - decoder_position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the - range `[0, config.max_position_embeddings - 1]`. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -OPT_ENCODE_INPUTS_DOCSTRING = r""" - Args: - input_ids (`jnp.ndarray` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide - it. - - Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, - config.max_position_embeddings - 1]`. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - -OPT_DECODE_INPUTS_DOCSTRING = r""" - Args: - decoder_input_ids (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - For translation and summarization training, `decoder_input_ids` should be provided. If no - `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right - for denoising pre-training following the paper. - encoder_outputs (`tuple(tuple(jnp.ndarray)`): - Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) - `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of - hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - encoder_attention_mask (`jnp.ndarray` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_attention_mask (`jnp.ndarray` of shape `(batch_size, target_sequence_length)`, *optional*): - Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also - be used by default. - - If you want to change padding behavior, you should modify to your needs. See diagram 1 in [the - paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. - decoder_position_ids (`numpy.ndarray` of shape `(batch_size, sequence_length)`, *optional*): - Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the - range `[0, config.max_position_embeddings - 1]`. - past_key_values (`Dict[str, np.ndarray]`, *optional*, returned by `init_cache` or when passing previous `past_key_values`): - Dictionary of pre-computed hidden-states (key and values in the attention blocks) that can be used for fast - auto-regressive decoding. Pre-computed key and value hidden-states are of shape *[batch_size, max_length]*. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. -""" - - -def shift_tokens_right(input_ids: np.array, pad_token_id: int, decoder_start_token_id: int) -> np.ndarray: - """ - Shift input ids one token to the right. - """ - shifted_input_ids = np.zeros_like(input_ids) - shifted_input_ids[:, 1:] = input_ids[:, :-1] - shifted_input_ids[:, 0] = decoder_start_token_id - - shifted_input_ids = np.where(shifted_input_ids == -100, pad_token_id, shifted_input_ids) - return shifted_input_ids - - -class FlaxOPTAttention(nn.Module): - config: OPTConfig - embed_dim: int - num_heads: int - dropout: float = 0.0 - causal: bool = False - bias: bool = True - dtype: jnp.dtype = jnp.float32 # the dtype of the computation - - def setup(self) -> None: - self.head_dim = self.embed_dim // self.num_heads - if self.head_dim * self.num_heads != self.embed_dim: - raise ValueError( - f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}" - f" and `num_heads`: {self.num_heads})." - ) - self.scaling = self.head_dim**-0.5 - dense = partial( - nn.Dense, - self.embed_dim, - use_bias=self.bias, - dtype=self.dtype, - kernel_init=jax.nn.initializers.normal(self.config.init_std), - ) - - self.q_proj, self.k_proj, self.v_proj = dense(), dense(), dense() - self.out_proj = dense() - - self.dropout_layer = nn.Dropout(rate=self.dropout) - - if self.causal: - self.causal_mask = make_causal_mask( - jnp.ones((1, self.config.max_position_embeddings), dtype="bool"), dtype="bool" - ) - - def _split_heads(self, hidden_states): - return hidden_states.reshape(hidden_states.shape[:2] + (self.num_heads, self.head_dim)) - - def _merge_heads(self, hidden_states): - return hidden_states.reshape(hidden_states.shape[:2] + (self.embed_dim,)) - - @nn.compact - def _concatenate_to_cache(self, key, value, query, attention_mask): - """ - This function takes projected key, value states from a single input token and concatenates the states to cached - states from previous steps. This function is slighly adapted from the official Flax repository: - https://github.com/google/flax/blob/491ce18759622506588784b4fca0e4bf05f8c8cd/flax/linen/attention.py#L252 - """ - # detect if we're initializing by absence of existing cache data. - is_initialized = self.has_variable("cache", "cached_key") - cached_key = self.variable("cache", "cached_key", jnp.zeros, key.shape, key.dtype) - cached_value = self.variable("cache", "cached_value", jnp.zeros, value.shape, value.dtype) - cache_index = self.variable("cache", "cache_index", lambda: jnp.array(0, dtype=jnp.int32)) - - if is_initialized: - *batch_dims, max_length, num_heads, depth_per_head = cached_key.value.shape - # update key, value caches with our new 1d spatial slices - cur_index = cache_index.value - indices = (0,) * len(batch_dims) + (cur_index, 0, 0) - key = lax.dynamic_update_slice(cached_key.value, key, indices) - value = lax.dynamic_update_slice(cached_value.value, value, indices) - cached_key.value = key - cached_value.value = value - num_updated_cache_vectors = query.shape[1] - cache_index.value = cache_index.value + num_updated_cache_vectors - # causal mask for cached decoder self-attention: our single query position should only attend to those key positions that have already been generated and cached, not the remaining zero elements. - pad_mask = jnp.broadcast_to( - jnp.arange(max_length) < cur_index + num_updated_cache_vectors, - tuple(batch_dims) + (1, num_updated_cache_vectors, max_length), - ) - attention_mask = combine_masks(pad_mask, attention_mask) - return key, value, attention_mask - - def __call__( - self, - hidden_states: jnp.ndarray, - key_value_states: Optional[jnp.ndarray] = None, - attention_mask: Optional[jnp.ndarray] = None, - init_cache: bool = False, - deterministic: bool = True, - ) -> Tuple[jnp.ndarray]: - """Input shape: Batch x Time x Channel""" - - # if key_value_states are provided this layer is used as a cross-attention layer - # for the decoder - is_cross_attention = key_value_states is not None - batch_size = hidden_states.shape[0] - - # get query proj - query_states = self.q_proj(hidden_states) - # get key, value proj - if is_cross_attention: - # cross_attentions - key_states = self.k_proj(key_value_states) - value_states = self.v_proj(key_value_states) - else: - # self_attention - key_states = self.k_proj(hidden_states) - value_states = self.v_proj(hidden_states) - - query_states = self._split_heads(query_states) - key_states = self._split_heads(key_states) - value_states = self._split_heads(value_states) - - # handle cache prepare causal attention mask - if self.causal: - query_length, key_length = query_states.shape[1], key_states.shape[1] - if self.has_variable("cache", "cached_key"): - mask_shift = self.variables["cache"]["cache_index"] - max_decoder_length = self.variables["cache"]["cached_key"].shape[1] - causal_mask = lax.dynamic_slice( - self.causal_mask, (0, 0, mask_shift, 0), (1, 1, query_length, max_decoder_length) - ) - else: - causal_mask = self.causal_mask[:, :, :query_length, :key_length] - causal_mask = jnp.broadcast_to(causal_mask, (batch_size,) + causal_mask.shape[1:]) - - # combine masks if needed - if attention_mask is not None and self.causal: - attention_mask = jnp.broadcast_to(jnp.expand_dims(attention_mask, axis=(-3, -2)), causal_mask.shape) - attention_mask = combine_masks(attention_mask, causal_mask) - elif self.causal: - attention_mask = causal_mask - elif attention_mask is not None: - attention_mask = jnp.expand_dims(attention_mask, axis=(-3, -2)) - - # During fast autoregressive decoding, we feed one position at a time, - # and cache the keys and values step by step. - if self.causal and (self.has_variable("cache", "cached_key") or init_cache): - key_states, value_states, attention_mask = self._concatenate_to_cache( - key_states, value_states, query_states, attention_mask - ) - - # Convert the boolean attention mask to an attention bias. - if attention_mask is not None: - # attention mask in the form of attention bias - attention_bias = lax.select( - attention_mask > 0, - jnp.full(attention_mask.shape, 0.0).astype(self.dtype), - jnp.full(attention_mask.shape, float("-inf")).astype(self.dtype), - ) - else: - attention_bias = None - - dropout_rng = None - if not deterministic and self.dropout > 0.0: - dropout_rng = self.make_rng("dropout") - - attn_weights = dot_product_attention_weights( - query_states, - key_states, - bias=attention_bias, - dropout_rng=dropout_rng, - dropout_rate=self.dropout, - broadcast_dropout=True, - deterministic=deterministic, - dtype=self.dtype, - precision=None, - ) - - attn_output = jnp.einsum("...hqk,...khd->...qhd", attn_weights, value_states) - attn_output = self._merge_heads(attn_output) - attn_output = self.out_proj(attn_output) - - return attn_output, attn_weights - - -# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartDecoderLayer with Bart->OPT -class FlaxOPTDecoderLayer(nn.Module): - config: OPTConfig - dtype: jnp.dtype = jnp.float32 - - def setup(self) -> None: - self.embed_dim = self.config.d_model - self.self_attn = FlaxOPTAttention( - config=self.config, - embed_dim=self.embed_dim, - num_heads=self.config.decoder_attention_heads, - dropout=self.config.attention_dropout, - causal=True, - dtype=self.dtype, - ) - self.dropout_layer = nn.Dropout(rate=self.config.dropout) - self.activation_fn = ACT2FN[self.config.activation_function] - self.activation_dropout_layer = nn.Dropout(rate=self.config.activation_dropout) - - self.self_attn_layer_norm = nn.LayerNorm(dtype=self.dtype, epsilon=1e-05) - self.encoder_attn = FlaxOPTAttention( - config=self.config, - embed_dim=self.embed_dim, - num_heads=self.config.decoder_attention_heads, - dropout=self.config.attention_dropout, - dtype=self.dtype, - ) - self.encoder_attn_layer_norm = nn.LayerNorm(dtype=self.dtype, epsilon=1e-05) - self.fc1 = nn.Dense( - self.config.encoder_ffn_dim, - dtype=self.dtype, - kernel_init=jax.nn.initializers.normal(self.config.init_std), - ) - self.fc2 = nn.Dense( - self.embed_dim, dtype=self.dtype, kernel_init=jax.nn.initializers.normal(self.config.init_std) - ) - self.final_layer_norm = nn.LayerNorm(dtype=self.dtype, epsilon=1e-05) - - def __call__( - self, - hidden_states: jnp.ndarray, - attention_mask: jnp.ndarray, - encoder_hidden_states: Optional[jnp.ndarray] = None, - encoder_attention_mask: Optional[jnp.ndarray] = None, - init_cache: bool = False, - output_attentions: bool = True, - deterministic: bool = True, - ) -> Tuple[jnp.ndarray]: - residual = hidden_states - - # Self Attention - hidden_states, self_attn_weights = self.self_attn( - hidden_states=hidden_states, attention_mask=attention_mask, init_cache=init_cache - ) - hidden_states = self.dropout_layer(hidden_states, deterministic=deterministic) - hidden_states = residual + hidden_states - hidden_states = self.self_attn_layer_norm(hidden_states) - - # Cross-Attention Block - cross_attn_weights = None - if encoder_hidden_states is not None: - residual = hidden_states - - hidden_states, cross_attn_weights = self.encoder_attn( - hidden_states=hidden_states, - key_value_states=encoder_hidden_states, - attention_mask=encoder_attention_mask, - ) - hidden_states = self.dropout_layer(hidden_states, deterministic=deterministic) - hidden_states = residual + hidden_states - hidden_states = self.encoder_attn_layer_norm(hidden_states) - - # Fully Connected - residual = hidden_states - hidden_states = self.activation_fn(self.fc1(hidden_states)) - hidden_states = self.activation_dropout_layer(hidden_states, deterministic=deterministic) - hidden_states = self.fc2(hidden_states) - hidden_states = self.dropout_layer(hidden_states, deterministic=deterministic) - hidden_states = residual + hidden_states - hidden_states = self.final_layer_norm(hidden_states) - - outputs = (hidden_states,) - - if output_attentions: - outputs += (self_attn_weights, cross_attn_weights) - - return outputs - - -# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartDecoderLayerCollection with Bart->OPT -class FlaxOPTDecoderLayerCollection(nn.Module): - config: OPTConfig - dtype: jnp.dtype = jnp.float32 # the dtype of the computation - - def setup(self): - self.layers = [ - FlaxOPTDecoderLayer(self.config, name=str(i), dtype=self.dtype) for i in range(self.config.decoder_layers) - ] - self.layerdrop = self.config.decoder_layerdrop - - def __call__( - self, - hidden_states, - attention_mask, - encoder_hidden_states: Optional[jnp.ndarray] = None, - encoder_attention_mask: Optional[jnp.ndarray] = None, - deterministic: bool = True, - init_cache: bool = False, - output_attentions: bool = False, - output_hidden_states: bool = False, - return_dict: bool = True, - ): - # decoder layers - all_hidden_states = () if output_hidden_states else None - all_self_attns = () if output_attentions else None - all_cross_attentions = () if (output_attentions and encoder_hidden_states is not None) else None - - for decoder_layer in self.layers: - if output_hidden_states: - all_hidden_states += (hidden_states,) - # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description) - dropout_probability = random.uniform(0, 1) - if not deterministic and (dropout_probability < self.layerdrop): - layer_outputs = (None, None, None) - else: - layer_outputs = decoder_layer( - hidden_states, - attention_mask=attention_mask, - encoder_hidden_states=encoder_hidden_states, - encoder_attention_mask=encoder_attention_mask, - init_cache=init_cache, - output_attentions=output_attentions, - deterministic=deterministic, - ) - - hidden_states = layer_outputs[0] - if output_attentions: - all_self_attns += (layer_outputs[1],) - - if encoder_hidden_states is not None: - all_cross_attentions += (layer_outputs[2],) - - # add hidden states from the last decoder layer - if output_hidden_states: - all_hidden_states += (hidden_states,) - - outputs = [hidden_states, all_hidden_states, all_self_attns, all_cross_attentions] - - if not return_dict: - return tuple(v for v in outputs if v is not None) - - return FlaxBaseModelOutputWithPastAndCrossAttentions( - last_hidden_state=hidden_states, - hidden_states=all_hidden_states, - attentions=all_self_attns, - cross_attentions=all_cross_attentions, - ) - - -# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartClassificationHead with Bart->OPT -class FlaxOPTClassificationHead(nn.Module): - """Head for sentence-level classification tasks.""" - - config: OPTConfig - inner_dim: int - num_classes: int - pooler_dropout: float - dtype: jnp.dtype = jnp.float32 - - def setup(self): - self.dense = nn.Dense( - self.inner_dim, dtype=self.dtype, kernel_init=jax.nn.initializers.normal(self.config.init_std) - ) - self.dropout = nn.Dropout(rate=self.pooler_dropout) - self.out_proj = nn.Dense( - self.num_classes, - dtype=self.dtype, - kernel_init=jax.nn.initializers.normal(self.config.init_std), - ) - - def __call__(self, hidden_states: jnp.ndarray, deterministic: bool): - hidden_states = self.dropout(hidden_states, deterministic=deterministic) - hidden_states = self.dense(hidden_states) - hidden_states = jnp.tanh(hidden_states) - hidden_states = self.dropout(hidden_states, deterministic=deterministic) - hidden_states = self.out_proj(hidden_states) - return hidden_states - - -# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartDecoder with Bart->OPT -class FlaxOPTDecoder(nn.Module): - config: OPTConfig - embed_tokens: nn.Embed - dtype: jnp.dtype = jnp.float32 # the dtype of the computation - - def setup(self): - self.dropout_layer = nn.Dropout(rate=self.config.dropout) - - embed_dim = self.config.d_model - self.padding_idx = self.config.pad_token_id - self.max_target_positions = self.config.max_position_embeddings - self.embed_scale = math.sqrt(self.config.d_model) if self.config.scale_embedding else 1.0 - - # OPT is set up so that if padding_idx is specified then offset the embedding ids by 2 - # and adjust num_embeddings appropriately. Other models don't have this hack - self.offset = 2 - self.embed_positions = nn.Embed( - self.config.max_position_embeddings + self.offset, - embed_dim, - embedding_init=jax.nn.initializers.normal(self.config.init_std), - ) - - self.layers = FlaxOPTDecoderLayerCollection(self.config, self.dtype) - self.layernorm_embedding = nn.LayerNorm(dtype=self.dtype, epsilon=1e-05) - - def __call__( - self, - input_ids, - attention_mask, - position_ids, - encoder_hidden_states: Optional[jnp.ndarray] = None, - encoder_attention_mask: Optional[jnp.ndarray] = None, - init_cache: bool = False, - output_attentions: bool = False, - output_hidden_states: bool = False, - return_dict: bool = True, - deterministic: bool = True, - ): - input_shape = input_ids.shape - input_ids = input_ids.reshape(-1, input_shape[-1]) - - inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale - - # embed positions - positions = self.embed_positions(position_ids + self.offset) - - hidden_states = inputs_embeds + positions - hidden_states = self.layernorm_embedding(hidden_states) - - hidden_states = self.dropout_layer(hidden_states, deterministic=deterministic) - - outputs = self.layers( - hidden_states, - attention_mask, - encoder_hidden_states, - encoder_attention_mask, - deterministic=deterministic, - init_cache=init_cache, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - ) - - if not return_dict: - return outputs - - return FlaxBaseModelOutputWithPastAndCrossAttentions( - last_hidden_state=outputs.last_hidden_state, - hidden_states=outputs.hidden_states, - attentions=outputs.attentions, - cross_attentions=outputs.cross_attentions, - ) - - -class FlaxOPTModule(nn.Module): - config: OPTConfig - dtype: jnp.dtype = jnp.float32 # the dtype of the computation - - def setup(self): - self.shared = nn.Embed( - self.config.vocab_size, - self.config.d_model, - embedding_init=jax.nn.initializers.normal(self.config.init_std), - ) - - self.decoder = FlaxOPTDecoder(self.config, dtype=self.dtype, embed_tokens=self.shared) - - def _get_decoder_module(self): - return self.decoder - - def __call__( - self, - input_ids, - attention_mask, - position_ids, - decoder_position_ids, - output_attentions: bool = False, - output_hidden_states: bool = False, - return_dict: bool = True, - deterministic: bool = True, - ): - - decoder_outputs = self.decoder( - input_ids=input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - deterministic=deterministic, - ) - - if not return_dict: - return decoder_outputs - - return FlaxSeq2SeqModelOutput( # TODO change model output - last_hidden_state=decoder_outputs.last_hidden_state, - decoder_hidden_states=decoder_outputs.hidden_states, - decoder_attentions=decoder_outputs.attentions, - cross_attentions=decoder_outputs.cross_attentions, - ) - - -# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartPreTrainedModel with BART->OPT,Bart->OPT,bart->opt -class FlaxOPTPreTrainedModel(FlaxPreTrainedModel): - config_class = OPTConfig - base_model_prefix: str = "model" - module_class: nn.Module = None - - def __init__( - self, - config: OPTConfig, - input_shape: Tuple[int] = (1, 1), - seed: int = 0, - dtype: jnp.dtype = jnp.float32, - _do_init: bool = True, - **kwargs - ): - module = self.module_class(config=config, dtype=dtype, **kwargs) - super().__init__(config, module, input_shape=input_shape, seed=seed, dtype=dtype, _do_init=_do_init) - - def init_weights(self, rng: jax.random.PRNGKey, input_shape: Tuple, params: FrozenDict = None) -> FrozenDict: - # init input tensors - input_ids = jnp.zeros(input_shape, dtype="i4") - # make sure initialization pass will work for FlaxOPTForSequenceClassificationModule - input_ids = input_ids.at[(..., -1)].set(self.config.eos_token_id) - attention_mask = jnp.ones_like(input_ids) - decoder_input_ids = input_ids - decoder_attention_mask = jnp.ones_like(input_ids) - - batch_size, sequence_length = input_ids.shape - position_ids = jnp.broadcast_to(jnp.arange(sequence_length)[None, :], (batch_size, sequence_length)) - decoder_position_ids = jnp.broadcast_to(jnp.arange(sequence_length)[None, :], (batch_size, sequence_length)) - - params_rng, dropout_rng = jax.random.split(rng) - rngs = {"params": params_rng, "dropout": dropout_rng} - - random_params = self.module.init( - rngs, - input_ids, - attention_mask, - decoder_input_ids, - decoder_attention_mask, - position_ids, - decoder_position_ids, - )["params"] - - if params is not None: - random_params = flatten_dict(unfreeze(random_params)) - params = flatten_dict(unfreeze(params)) - for missing_key in self._missing_keys: - params[missing_key] = random_params[missing_key] - self._missing_keys = set() - return freeze(unflatten_dict(params)) - else: - return random_params - - def init_cache(self, batch_size, max_length, encoder_outputs): - r""" - Args: - batch_size (`int`): - batch_size used for fast auto-regressive decoding. Defines the batch size of the initialized cache. - max_length (`int`): - maximum possible length for auto-regressive decoding. Defines the sequence length of the initialized - cache. - encoder_outputs (`Union[FlaxBaseModelOutput, tuple(tuple(jnp.ndarray)]`): - `encoder_outputs` consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: - `attentions`). `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) - is a sequence of hidden-states at the output of the last layer of the encoder. Used in the - cross-attention of the decoder. - """ - # init input variables to retrieve cache - decoder_input_ids = jnp.ones((batch_size, max_length), dtype="i4") - decoder_attention_mask = jnp.ones_like(decoder_input_ids) - decoder_position_ids = jnp.broadcast_to( - jnp.arange(jnp.atleast_2d(decoder_input_ids).shape[-1]), decoder_input_ids.shape - ) - - def _decoder_forward(module, decoder_input_ids, decoder_attention_mask, decoder_position_ids, **kwargs): - decoder_module = module._get_decoder_module() - return decoder_module( - decoder_input_ids, - decoder_attention_mask, - decoder_position_ids, - **kwargs, - ) - - init_variables = self.module.init( - jax.random.PRNGKey(0), - decoder_input_ids=decoder_input_ids, - decoder_attention_mask=decoder_attention_mask, - decoder_position_ids=decoder_position_ids, - encoder_hidden_states=encoder_outputs[0], - init_cache=True, - method=_decoder_forward, # we only need to call the decoder to init the cache - ) - return unfreeze(init_variables["cache"]) - - @add_start_docstrings(OPT_ENCODE_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=FlaxBaseModelOutput, config_class=OPTConfig) - def encode( - self, - input_ids: jnp.ndarray, - attention_mask: Optional[jnp.ndarray] = None, - position_ids: Optional[jnp.ndarray] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - train: bool = False, - params: dict = None, - dropout_rng: PRNGKey = None, - ): - r""" - Returns: - - Example: - - ```python - >>> from transformers import OPTTokenizer, FlaxOPTForConditionalGeneration - - >>> model = FlaxOPTForConditionalGeneration.from_pretrained("facebook/opt-large-cnn") - >>> tokenizer = OPTTokenizer.from_pretrained("facebook/opt-large-cnn") - - >>> text = "My friends are cool but they eat too many carbs." - >>> inputs = tokenizer(text, max_length=1024, return_tensors="jax") - >>> encoder_outputs = model.encode(**inputs) - ```""" - output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions - output_hidden_states = ( - output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states - ) - return_dict = return_dict if return_dict is not None else self.config.return_dict - - if attention_mask is None: - attention_mask = jnp.ones_like(input_ids) - if position_ids is None: - batch_size, sequence_length = input_ids.shape - position_ids = jnp.broadcast_to(jnp.arange(sequence_length)[None, :], (batch_size, sequence_length)) - - # Handle any PRNG if needed - rngs = {} - if dropout_rng is not None: - rngs["dropout"] = dropout_rng - - def _encoder_forward(module, input_ids, attention_mask, position_ids, **kwargs): - encode_module = module._get_encoder_module() - return encode_module(input_ids, attention_mask, position_ids, **kwargs) - - return self.module.apply( - {"params": params or self.params}, - input_ids=jnp.array(input_ids, dtype="i4"), - attention_mask=jnp.array(attention_mask, dtype="i4"), - position_ids=jnp.array(position_ids, dtype="i4"), - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - deterministic=not train, - rngs=rngs, - method=_encoder_forward, - ) - - @add_start_docstrings(OPT_DECODE_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=FlaxBaseModelOutputWithPastAndCrossAttentions, config_class=OPTConfig) - def decode( - self, - decoder_input_ids, - encoder_outputs, - encoder_attention_mask: Optional[jnp.ndarray] = None, - decoder_attention_mask: Optional[jnp.ndarray] = None, - decoder_position_ids: Optional[jnp.ndarray] = None, - past_key_values: dict = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - train: bool = False, - params: dict = None, - dropout_rng: PRNGKey = None, - ): - r""" - Returns: - - Example: - - ```python - >>> import jax.numpy as jnp - >>> from transformers import OPTTokenizer, FlaxOPTForConditionalGeneration - - >>> model = FlaxOPTForConditionalGeneration.from_pretrained("facebook/opt-large-cnn") - >>> tokenizer = OPTTokenizer.from_pretrained("facebook/opt-large-cnn") - - >>> text = "My friends are cool but they eat too many carbs." - >>> inputs = tokenizer(text, max_length=1024, return_tensors="jax") - >>> encoder_outputs = model.encode(**inputs) - - >>> decoder_start_token_id = model.config.decoder_start_token_id - >>> decoder_input_ids = jnp.ones((inputs.input_ids.shape[0], 1), dtype="i4") * decoder_start_token_id - - >>> outputs = model.decode(decoder_input_ids, encoder_outputs) - >>> last_decoder_hidden_states = outputs.last_hidden_state - ```""" - output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions - output_hidden_states = ( - output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states - ) - return_dict = return_dict if return_dict is not None else self.config.return_dict - - encoder_hidden_states = encoder_outputs[0] - if encoder_attention_mask is None: - batch_size, sequence_length = encoder_hidden_states.shape[:2] - encoder_attention_mask = jnp.ones((batch_size, sequence_length)) - - batch_size, sequence_length = decoder_input_ids.shape - if decoder_attention_mask is None: - decoder_attention_mask = jnp.ones((batch_size, sequence_length)) - - if decoder_position_ids is None: - if past_key_values is not None: - raise ValueError("Make sure to provide `decoder_position_ids` when passing `past_key_values`.") - - decoder_position_ids = jnp.broadcast_to( - jnp.arange(sequence_length)[None, :], (batch_size, sequence_length) - ) - - # Handle any PRNG if needed - rngs = {} - if dropout_rng is not None: - rngs["dropout"] = dropout_rng - - inputs = {"params": params or self.params} - - # if past_key_values are passed then cache is already initialized a private flag init_cache has to be - # passed down to ensure cache is used. It has to be made sure that cache is marked as mutable so that - # it can be changed by FlaxOPTAttention module - if past_key_values: - inputs["cache"] = past_key_values - mutable = ["cache"] - else: - mutable = False - - def _decoder_forward(module, decoder_input_ids, decoder_attention_mask, decoder_position_ids, **kwargs): - decoder_module = module._get_decoder_module() - return decoder_module( - decoder_input_ids, - decoder_attention_mask, - decoder_position_ids, - **kwargs, - ) - - outputs = self.module.apply( - inputs, - decoder_input_ids=jnp.array(decoder_input_ids, dtype="i4"), - decoder_attention_mask=jnp.array(decoder_attention_mask, dtype="i4"), - decoder_position_ids=jnp.array(decoder_position_ids, dtype="i4"), - encoder_hidden_states=encoder_hidden_states, - encoder_attention_mask=jnp.array(encoder_attention_mask, dtype="i4"), - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - deterministic=not train, - rngs=rngs, - mutable=mutable, - method=_decoder_forward, - ) - - # add updated cache to model output - if past_key_values is not None and return_dict: - outputs, past = outputs - outputs["past_key_values"] = unfreeze(past["cache"]) - return outputs - elif past_key_values is not None and not return_dict: - outputs, past = outputs - outputs = outputs[:1] + (unfreeze(past["cache"]),) + outputs[1:] - - return outputs - - @add_start_docstrings_to_model_forward(OPT_INPUTS_DOCSTRING) - def __call__( - self, - input_ids: jnp.ndarray, - attention_mask: Optional[jnp.ndarray] = None, - decoder_input_ids: Optional[jnp.ndarray] = None, - decoder_attention_mask: Optional[jnp.ndarray] = None, - position_ids: Optional[jnp.ndarray] = None, - decoder_position_ids: Optional[jnp.ndarray] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - train: bool = False, - params: dict = None, - dropout_rng: PRNGKey = None, - ): - output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions - output_hidden_states = ( - output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states - ) - return_dict = return_dict if return_dict is not None else self.config.return_dict - - # prepare encoder inputs - if attention_mask is None: - attention_mask = jnp.ones_like(input_ids) - if position_ids is None: - batch_size, sequence_length = input_ids.shape - position_ids = jnp.broadcast_to(jnp.arange(sequence_length)[None, :], (batch_size, sequence_length)) - - # prepare decoder inputs - if decoder_input_ids is None: - decoder_input_ids = shift_tokens_right( - input_ids, self.config.pad_token_id, decoder_start_token_id=self.config.decoder_start_token_id - ) - if decoder_attention_mask is None: - decoder_attention_mask = jnp.ones_like(decoder_input_ids) - if decoder_position_ids is None: - batch_size, sequence_length = decoder_input_ids.shape - decoder_position_ids = jnp.broadcast_to( - jnp.arange(sequence_length)[None, :], (batch_size, sequence_length) - ) - - # Handle any PRNG if needed - rngs = {"dropout": dropout_rng} if dropout_rng is not None else {} - - return self.module.apply( - {"params": params or self.params}, - input_ids=jnp.array(input_ids, dtype="i4"), - attention_mask=jnp.array(attention_mask, dtype="i4"), - position_ids=jnp.array(position_ids, dtype="i4"), - decoder_input_ids=jnp.array(decoder_input_ids, dtype="i4"), - decoder_attention_mask=jnp.array(decoder_attention_mask, dtype="i4"), - decoder_position_ids=jnp.array(decoder_position_ids, dtype="i4"), - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - deterministic=not train, - rngs=rngs, - ) - - -@add_start_docstrings( - "The bare OPT Model transformer outputting raw hidden-states without any specific head on top.", - OPT_START_DOCSTRING, -) -# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartModel with Bart->OPT -class FlaxOPTModel(FlaxOPTPreTrainedModel): - config: OPTConfig - dtype: jnp.dtype = jnp.float32 # the dtype of the computation - module_class = FlaxOPTModule - - -append_call_sample_docstring( - FlaxOPTModel, _TOKENIZER_FOR_DOC, _CHECKPOINT_FOR_DOC, FlaxSeq2SeqModelOutput, _CONFIG_FOR_DOC -) - - -# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartDecoderPreTrainedModel with BART->OPT,Bart->OPT -class FlaxOPTDecoderPreTrainedModel(FlaxPreTrainedModel): - config_class = OPTConfig - base_model_prefix: str = "model" - module_class: nn.Module = None - - def __init__( - self, - config: OPTConfig, - input_shape: Tuple[int] = (1, 1), - seed: int = 0, - dtype: jnp.dtype = jnp.float32, - _do_init: bool = True, - **kwargs - ): - config.is_decoder = True - config.is_encoder_decoder = False - module = self.module_class(config=config, dtype=dtype, **kwargs) - super().__init__(config, module, input_shape=input_shape, seed=seed, dtype=dtype, _do_init=_do_init) - - def init_weights(self, rng: jax.random.PRNGKey, input_shape: Tuple, params: FrozenDict = None) -> FrozenDict: - # init input tensors - input_ids = jnp.zeros(input_shape, dtype="i4") - attention_mask = jnp.ones_like(input_ids) - - batch_size, sequence_length = input_ids.shape - position_ids = jnp.broadcast_to(jnp.arange(sequence_length)[None, :], (batch_size, sequence_length)) - - params_rng, dropout_rng = jax.random.split(rng) - rngs = {"params": params_rng, "dropout": dropout_rng} - encoder_hidden_states = jnp.zeros(input_shape + (self.config.d_model,)) - encoder_attention_mask = attention_mask - module_init_outputs = self.module.init( - rngs, - input_ids, - attention_mask, - position_ids, - encoder_hidden_states, - encoder_attention_mask, - return_dict=False, - ) - return module_init_outputs["params"] - - def init_cache(self, batch_size, max_length): - r""" - Args: - batch_size (`int`): - batch_size used for fast auto-regressive decoding. Defines the batch size of the initialized cache. - max_length (`int`): - maximum possible length for auto-regressive decoding. Defines the sequence length of the initialized - cache. - """ - # init input variables to retrieve cache - input_ids = jnp.ones((batch_size, max_length), dtype="i4") - attention_mask = jnp.ones_like(input_ids, dtype="i4") - position_ids = jnp.broadcast_to(jnp.arange(jnp.atleast_2d(input_ids).shape[-1]), input_ids.shape) - - init_variables = self.module.init( - jax.random.PRNGKey(0), input_ids, attention_mask, position_ids, return_dict=False, init_cache=True - ) - return unfreeze(init_variables["cache"]) - - @add_start_docstrings_to_model_forward(OPT_DECODE_INPUTS_DOCSTRING) - def __call__( - self, - input_ids: jnp.ndarray, - attention_mask: Optional[jnp.ndarray] = None, - position_ids: Optional[jnp.ndarray] = None, - encoder_hidden_states: Optional[jnp.ndarray] = None, - encoder_attention_mask: Optional[jnp.ndarray] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - train: bool = False, - params: dict = None, - past_key_values: dict = None, - dropout_rng: PRNGKey = None, - ): - output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions - output_hidden_states = ( - output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states - ) - return_dict = return_dict if return_dict is not None else self.config.return_dict - - if encoder_hidden_states is not None and encoder_attention_mask is None: - batch_size, sequence_length = encoder_hidden_states.shape[:2] - encoder_attention_mask = jnp.ones((batch_size, sequence_length)) - - # prepare decoder inputs - if attention_mask is None: - attention_mask = jnp.ones_like(input_ids) - if position_ids is None: - batch_size, sequence_length = input_ids.shape - position_ids = jnp.broadcast_to(jnp.arange(sequence_length)[None, :], (batch_size, sequence_length)) - - # Handle any PRNG if needed - rngs = {"dropout": dropout_rng} if dropout_rng is not None else {} - - inputs = {"params": params or self.params} - - # if past_key_values are passed then cache is already initialized a private flag init_cache has to be passed - # down to ensure cache is used. It has to be made sure that cache is marked as mutable so that it can be - # changed by FlaxOPTAttention module - if past_key_values: - inputs["cache"] = past_key_values - mutable = ["cache"] - else: - mutable = False - - outputs = self.module.apply( - inputs, - input_ids=jnp.array(input_ids, dtype="i4"), - attention_mask=jnp.array(attention_mask, dtype="i4"), - position_ids=jnp.array(position_ids, dtype="i4"), - encoder_hidden_states=encoder_hidden_states, - encoder_attention_mask=encoder_attention_mask, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - deterministic=not train, - rngs=rngs, - mutable=mutable, - ) - - # add updated cache to model output - if past_key_values is not None and return_dict: - outputs, past_key_values = outputs - outputs["past_key_values"] = unfreeze(past_key_values["cache"]) - return outputs - elif past_key_values is not None and not return_dict: - outputs, past_key_values = outputs - outputs = outputs[:1] + (unfreeze(past_key_values["cache"]),) + outputs[1:] - - return outputs - - -# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartDecoderWrapper with Bart->OPT -class FlaxOPTDecoderWrapper(nn.Module): - """ - This wrapper class is a helper class to correctly load pretrained checkpoints when the causal language model is - used in combination with the [`EncoderDecoderModel`] framework. - """ - - config: OPTConfig - dtype: jnp.dtype = jnp.float32 - - def setup(self): - embed_dim = self.config.d_model - embed_tokens = nn.Embed( - self.config.vocab_size, - embed_dim, - embedding_init=jax.nn.initializers.normal(self.config.init_std), - ) - self.decoder = FlaxOPTDecoder(config=self.config, embed_tokens=embed_tokens, dtype=self.dtype) - - def __call__(self, *args, **kwargs): - return self.decoder(*args, **kwargs) - - -# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartForCausalLMModule with Bart->OPT -class FlaxOPTForCausalLMModule(nn.Module): - config: OPTConfig - dtype: jnp.dtype = jnp.float32 - - def setup(self): - self.model = FlaxOPTDecoderWrapper(config=self.config, dtype=self.dtype) - self.lm_head = nn.Dense( - self.config.vocab_size, - use_bias=False, - dtype=self.dtype, - kernel_init=jax.nn.initializers.normal(self.config.init_std), - ) - - def __call__( - self, - input_ids, - attention_mask, - position_ids, - encoder_hidden_states: Optional[jnp.ndarray] = None, - encoder_attention_mask: Optional[jnp.ndarray] = None, - init_cache: bool = False, - output_attentions: bool = False, - output_hidden_states: bool = False, - return_dict: bool = True, - deterministic: bool = True, - ): - - outputs = self.model( - input_ids, - attention_mask, - position_ids, - encoder_hidden_states, - encoder_attention_mask, - deterministic=deterministic, - init_cache=init_cache, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - ) - - hidden_states = outputs[0] - - if self.config.tie_word_embeddings: - shared_embedding = self.model.variables["params"]["decoder"]["embed_tokens"]["embedding"] - lm_logits = self.lm_head.apply({"params": {"kernel": shared_embedding.T}}, hidden_states) - else: - lm_logits = self.lm_head(hidden_states) - - if not return_dict: - return (lm_logits,) + outputs[1:] - - return FlaxCausalLMOutputWithCrossAttentions( - logits=lm_logits, - hidden_states=outputs.hidden_states, - attentions=outputs.attentions, - cross_attentions=outputs.cross_attentions, - ) - - -@add_start_docstrings( - """ - OPT Decoder Model with a language modeling head on top (linear layer with weights tied to the input embeddings) e.g - for autoregressive tasks. - """, - OPT_START_DOCSTRING, -) -# Copied from transformers.models.bart.modeling_flax_bart.FlaxBartForCausalLM with Bart->OPT -class FlaxOPTForCausalLM(FlaxOPTDecoderPreTrainedModel): - module_class = FlaxOPTForCausalLMModule - - def prepare_inputs_for_generation(self, input_ids, max_length, attention_mask: Optional[jnp.DeviceArray] = None): - # initializing the cache - batch_size, seq_length = input_ids.shape - - past_key_values = self.init_cache(batch_size, max_length) - # Note that usually one would have to put 0's in the attention_mask for x > input_ids.shape[-1] and x < cache_length. - # But since the decoder uses a causal mask, those positions are masked anyway. - # Thus, we can create a single static attention_mask here, which is more efficient for compilation - extended_attention_mask = jnp.ones((batch_size, max_length), dtype="i4") - if attention_mask is not None: - position_ids = attention_mask.cumsum(axis=-1) - 1 - extended_attention_mask = lax.dynamic_update_slice(extended_attention_mask, attention_mask, (0, 0)) - else: - position_ids = jnp.broadcast_to(jnp.arange(seq_length, dtype="i4")[None, :], (batch_size, seq_length)) - - return { - "past_key_values": past_key_values, - "attention_mask": extended_attention_mask, - "position_ids": position_ids, - } - - def update_inputs_for_generation(self, model_outputs, model_kwargs): - model_kwargs["past_key_values"] = model_outputs.past_key_values - model_kwargs["position_ids"] = model_kwargs["position_ids"][:, -1:] + 1 - return model_kwargs - - -append_call_sample_docstring( - FlaxOPTForCausalLM, - _TOKENIZER_FOR_DOC, - _CHECKPOINT_FOR_DOC, - FlaxCausalLMOutputWithCrossAttentions, - _CONFIG_FOR_DOC, -) diff --git a/src/transformers/models/opt/modeling_tf_opt.py b/src/transformers/models/opt/modeling_tf_opt.py deleted file mode 100644 index e2685799ddad0..0000000000000 --- a/src/transformers/models/opt/modeling_tf_opt.py +++ /dev/null @@ -1,1215 +0,0 @@ -# coding=utf-8 -# Copyright 2021 The Fairseq Authors and The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" TF 2.0 OPT model.""" - - -import random -from typing import Optional, Tuple, Union - -import numpy as np -import tensorflow as tf - -from ...activations_tf import get_tf_activation -from ...modeling_tf_outputs import TFBaseModelOutput, TFBaseModelOutputWithPastAndCrossAttentions, TFSeq2SeqModelOutput - -# Public API -from ...modeling_tf_utils import ( - DUMMY_INPUTS, - TFModelInputType, - TFPreTrainedModel, - TFSharedEmbeddings, - TFWrappedEmbeddings, - keras_serializable, - unpack_inputs, -) -from ...tf_utils import shape_list, stable_softmax -from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging -from .configuration_opt import OPTConfig - - -logger = logging.get_logger(__name__) - -_CHECKPOINT_FOR_DOC = "facebook/bart-large" -_CONFIG_FOR_DOC = "OPTConfig" -_TOKENIZER_FOR_DOC = "GPT2Tokenizer" - - -LARGE_NEGATIVE = -1e8 - - -def shift_tokens_right(input_ids: tf.Tensor, pad_token_id: int, decoder_start_token_id: int): - pad_token_id = tf.cast(pad_token_id, input_ids.dtype) - decoder_start_token_id = tf.cast(decoder_start_token_id, input_ids.dtype) - start_tokens = tf.fill((shape_list(input_ids)[0], 1), decoder_start_token_id) - shifted_input_ids = tf.concat([start_tokens, input_ids[:, :-1]], -1) - # replace possible -100 values in labels by `pad_token_id` - shifted_input_ids = tf.where( - shifted_input_ids == -100, tf.fill(shape_list(shifted_input_ids), pad_token_id), shifted_input_ids - ) - - if tf.executing_eagerly(): - # "Verify that `labels` has only positive values and -100" - assert_gte0 = tf.debugging.assert_greater_equal(shifted_input_ids, tf.constant(0, dtype=input_ids.dtype)) - - # Make sure the assertion op is called by wrapping the result in an identity no-op - with tf.control_dependencies([assert_gte0]): - shifted_input_ids = tf.identity(shifted_input_ids) - - return shifted_input_ids - - -def _make_causal_mask(input_ids_shape: tf.TensorShape, past_key_values_length: int = 0): - """ - Make causal mask used for bi-directional self-attention. - """ - bsz, tgt_len = input_ids_shape - mask = tf.ones((tgt_len, tgt_len)) * LARGE_NEGATIVE - mask_cond = tf.range(shape_list(mask)[-1]) - - mask = tf.where(mask_cond < tf.reshape(mask_cond + 1, (shape_list(mask)[-1], 1)), 0.0, mask) - - if past_key_values_length > 0: - mask = tf.concat([tf.zeros((tgt_len, past_key_values_length)), mask], axis=-1) - - return tf.tile(mask[None, None, :, :], (bsz, 1, 1, 1)) - - -def _expand_mask(mask: tf.Tensor, tgt_len: Optional[int] = None, past_key_values_length: int = 0): - """ - Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. - """ - src_len = shape_list(mask)[1] - tgt_len = tgt_len if tgt_len is not None else src_len - one_cst = tf.constant(1.0) - mask = tf.cast(mask, dtype=one_cst.dtype) - expanded_mask = tf.tile(mask[:, None, None, :], (1, 1, tgt_len, 1)) - - return (one_cst - expanded_mask) * LARGE_NEGATIVE - - -# Copied from transformers.models.bart.modeling_tf_bart.TFBartLearnedPositionalEmbedding with Bart->OPT -class TFOPTLearnedPositionalEmbedding(TFSharedEmbeddings): - """ - This module learns positional embeddings up to a fixed maximum size. - """ - - def __init__(self, num_embeddings: int, embedding_dim: int, **kwargs): - # OPT is set up so that if padding_idx is specified then offset the embedding ids by 2 - # and adjust num_embeddings appropriately. Other models don't have this hack - self.offset = 2 - super().__init__(num_embeddings + self.offset, embedding_dim, **kwargs) - - def call(self, input_shape: tf.TensorShape, past_key_values_length: int = 0): - """Input is expected to be of size [bsz x seqlen].""" - bsz, seq_len = input_shape[:2] - - positions = tf.range(past_key_values_length, seq_len + past_key_values_length, delta=1, name="range") - return super().call(positions + self.offset) - - -# Copied from transformers.models.bart.modeling_tf_bart.TFBartAttention with Bart->OPT -class TFOPTAttention(tf.keras.layers.Layer): - """Multi-headed attention from "Attention Is All You Need""" - - def __init__( - self, - embed_dim: int, - num_heads: int, - dropout: float = 0.0, - is_decoder: bool = False, - bias: bool = True, - **kwargs, - ): - super().__init__(**kwargs) - self.embed_dim = embed_dim - - self.num_heads = num_heads - self.dropout = tf.keras.layers.Dropout(dropout) - self.head_dim = embed_dim // num_heads - if (self.head_dim * num_heads) != self.embed_dim: - raise ValueError( - f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim}" - f" and `num_heads`: {num_heads})." - ) - self.scaling = self.head_dim**-0.5 - self.is_decoder = is_decoder - - self.k_proj = tf.keras.layers.Dense(embed_dim, use_bias=bias, name="k_proj") - self.q_proj = tf.keras.layers.Dense(embed_dim, use_bias=bias, name="q_proj") - self.v_proj = tf.keras.layers.Dense(embed_dim, use_bias=bias, name="v_proj") - self.out_proj = tf.keras.layers.Dense(embed_dim, use_bias=bias, name="out_proj") - - def _shape(self, tensor: tf.Tensor, seq_len: int, bsz: int): - return tf.transpose(tf.reshape(tensor, (bsz, seq_len, self.num_heads, self.head_dim)), (0, 2, 1, 3)) - - def call( - self, - hidden_states: tf.Tensor, - key_value_states: Optional[tf.Tensor] = None, - past_key_value: Optional[Tuple[Tuple[tf.Tensor]]] = None, - attention_mask: Optional[tf.Tensor] = None, - layer_head_mask: Optional[tf.Tensor] = None, - training: Optional[bool] = False, - ) -> Tuple[tf.Tensor, Optional[tf.Tensor]]: - """Input shape: Batch x Time x Channel""" - - # if key_value_states are provided this layer is used as a cross-attention layer - # for the decoder - is_cross_attention = key_value_states is not None - bsz, tgt_len, embed_dim = shape_list(hidden_states) - - # get query proj - query_states = self.q_proj(hidden_states) * self.scaling - # get key, value proj - if is_cross_attention and past_key_value is not None: - # reuse k,v, cross_attentions - key_states = past_key_value[0] - value_states = past_key_value[1] - elif is_cross_attention: - # cross_attentions - key_states = self._shape(self.k_proj(key_value_states), -1, bsz) - value_states = self._shape(self.v_proj(key_value_states), -1, bsz) - elif past_key_value is not None: - # reuse k, v, self_attention - key_states = self._shape(self.k_proj(hidden_states), -1, bsz) - value_states = self._shape(self.v_proj(hidden_states), -1, bsz) - key_states = tf.concat([past_key_value[0], key_states], axis=2) - value_states = tf.concat([past_key_value[1], value_states], axis=2) - else: - # self_attention - key_states = self._shape(self.k_proj(hidden_states), -1, bsz) - value_states = self._shape(self.v_proj(hidden_states), -1, bsz) - - if self.is_decoder: - # if cross_attention save Tuple(tf.Tensor, tf.Tensor) of all cross attention key/value_states. - # Further calls to cross_attention layer can then reuse all cross-attention - # key/value_states (first "if" case) - # if uni-directional self-attention (decoder) save Tuple(tf.Tensor, tf.Tensor) of - # all previous decoder key/value_states. Further calls to uni-directional self-attention - # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) - # if encoder bi-directional self-attention `past_key_value` is always `None` - past_key_value = (key_states, value_states) - - proj_shape = (bsz * self.num_heads, -1, self.head_dim) - query_states = tf.reshape(self._shape(query_states, tgt_len, bsz), proj_shape) - key_states = tf.reshape(key_states, proj_shape) - value_states = tf.reshape(value_states, proj_shape) - - src_len = shape_list(key_states)[1] - attn_weights = tf.matmul(query_states, key_states, transpose_b=True) - - # The tf.debugging asserts are not compliant with XLA then they - # have to be disabled in other modes than eager. - if tf.executing_eagerly(): - tf.debugging.assert_equal( - shape_list(attn_weights), - [bsz * self.num_heads, tgt_len, src_len], - message=f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is {shape_list(attn_weights)}", - ) - - if attention_mask is not None: - # The tf.debugging asserts are not compliant with XLA then they - # have to be disabled in other modes than eager. - if tf.executing_eagerly(): - tf.debugging.assert_equal( - shape_list(attention_mask), - [bsz, 1, tgt_len, src_len], - message=f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {shape_list(attention_mask)}", - ) - - attention_mask = tf.cast(attention_mask, dtype=attn_weights.dtype) - attn_weights = tf.reshape(attn_weights, (bsz, self.num_heads, tgt_len, src_len)) + attention_mask - attn_weights = tf.reshape(attn_weights, (bsz * self.num_heads, tgt_len, src_len)) - - attn_weights = stable_softmax(attn_weights, axis=-1) - - if layer_head_mask is not None: - # The tf.debugging asserts are not compliant with XLA then they - # have to be disabled in other modes than eager. - if tf.executing_eagerly(): - tf.debugging.assert_equal( - shape_list(layer_head_mask), - [self.num_heads], - message=f"Head mask for a single layer should be of size {(self.num_heads)}, but is {shape_list(layer_head_mask)}", - ) - - attn_weights = tf.reshape(layer_head_mask, (1, -1, 1, 1)) * tf.reshape( - attn_weights, (bsz, self.num_heads, tgt_len, src_len) - ) - attn_weights = tf.reshape(attn_weights, (bsz * self.num_heads, tgt_len, src_len)) - - attn_probs = self.dropout(attn_weights, training=training) - attn_output = tf.matmul(attn_probs, value_states) - - # The tf.debugging asserts are not compliant with XLA then they - # have to be disabled in other modes than eager. - if tf.executing_eagerly(): - tf.debugging.assert_equal( - shape_list(attn_output), - [bsz * self.num_heads, tgt_len, self.head_dim], - message=f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is {shape_list(attn_output)}", - ) - - attn_output = tf.transpose( - tf.reshape(attn_output, (bsz, self.num_heads, tgt_len, self.head_dim)), (0, 2, 1, 3) - ) - attn_output = tf.reshape(attn_output, (bsz, tgt_len, embed_dim)) - - attn_output = self.out_proj(attn_output) - attn_weights: tf.Tensor = tf.reshape(attn_weights, (bsz, self.num_heads, tgt_len, src_len)) - - return attn_output, attn_weights, past_key_value - - -# Copied from transformers.models.bart.modeling_tf_bart.TFBartEncoderLayer with Bart->OPT -class TFOPTEncoderLayer(tf.keras.layers.Layer): - def __init__(self, config: OPTConfig, **kwargs): - super().__init__(**kwargs) - self.embed_dim = config.d_model - self.self_attn = TFOPTAttention( - self.embed_dim, config.encoder_attention_heads, dropout=config.attention_dropout, name="self_attn" - ) - self.self_attn_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="self_attn_layer_norm") - self.dropout = tf.keras.layers.Dropout(config.dropout) - self.activation_fn = get_tf_activation(config.activation_function) - self.activation_dropout = tf.keras.layers.Dropout(config.activation_dropout) - self.fc1 = tf.keras.layers.Dense(config.encoder_ffn_dim, name="fc1") - self.fc2 = tf.keras.layers.Dense(self.embed_dim, name="fc2") - self.final_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="final_layer_norm") - - def call( - self, - hidden_states: tf.Tensor, - attention_mask: Optional[Union[np.ndarray, tf.Tensor]], - layer_head_mask: Optional[tf.Tensor], - training: Optional[bool] = False, - ) -> tf.Tensor: - """ - Args: - hidden_states (`tf.Tensor`): input to the layer of shape `(seq_len, batch, embed_dim)` - attention_mask (`tf.Tensor`): attention mask of size - `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. - layer_head_mask (`tf.Tensor`): mask for attention heads in a given layer of size - `(encoder_attention_heads,)` - """ - residual = hidden_states - hidden_states, self_attn_weights, _ = self.self_attn( - hidden_states=hidden_states, attention_mask=attention_mask, layer_head_mask=layer_head_mask - ) - - # The tf.debugging asserts are not compliant with XLA then they - # have to be disabled in other modes than eager. - if tf.executing_eagerly(): - tf.debugging.assert_equal( - shape_list(hidden_states), - shape_list(residual), - message=f"Self attn modified the shape of query {shape_list(residual)} to {shape_list(hidden_states)}", - ) - - hidden_states = self.dropout(hidden_states, training=training) - hidden_states = residual + hidden_states - hidden_states = self.self_attn_layer_norm(hidden_states) - - residual = hidden_states - hidden_states = self.activation_fn(self.fc1(hidden_states)) - hidden_states = self.activation_dropout(hidden_states, training=training) - hidden_states = self.fc2(hidden_states) - hidden_states = self.dropout(hidden_states, training=training) - hidden_states = residual + hidden_states - hidden_states = self.final_layer_norm(hidden_states) - - return hidden_states, self_attn_weights - - -# Copied from transformers.models.bart.modeling_tf_bart.TFBartDecoderLayer with Bart->OPT -class TFOPTDecoderLayer(tf.keras.layers.Layer): - def __init__(self, config: OPTConfig, **kwargs): - super().__init__(**kwargs) - self.embed_dim = config.d_model - self.self_attn = TFOPTAttention( - embed_dim=self.embed_dim, - num_heads=config.decoder_attention_heads, - dropout=config.attention_dropout, - name="self_attn", - is_decoder=True, - ) - self.dropout = tf.keras.layers.Dropout(config.dropout) - self.activation_fn = get_tf_activation(config.activation_function) - self.activation_dropout = tf.keras.layers.Dropout(config.activation_dropout) - - self.self_attn_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="self_attn_layer_norm") - self.encoder_attn = TFOPTAttention( - self.embed_dim, - config.decoder_attention_heads, - dropout=config.attention_dropout, - name="encoder_attn", - is_decoder=True, - ) - self.encoder_attn_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="encoder_attn_layer_norm") - self.fc1 = tf.keras.layers.Dense(config.decoder_ffn_dim, name="fc1") - self.fc2 = tf.keras.layers.Dense(self.embed_dim, name="fc2") - self.final_layer_norm = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="final_layer_norm") - - def call( - self, - hidden_states: tf.Tensor, - attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, - encoder_hidden_states: Optional[Union[np.ndarray, tf.Tensor]] = None, - encoder_attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, - layer_head_mask: Optional[tf.Tensor] = None, - cross_attn_layer_head_mask: Optional[tf.Tensor] = None, - past_key_value: Optional[Tuple[Tuple[Union[np.ndarray, tf.Tensor]]]] = None, - training: Optional[bool] = False, - ) -> Tuple[tf.Tensor, tf.Tensor, Tuple[Tuple[tf.Tensor]]]: - """ - Args: - hidden_states (`tf.Tensor`): input to the layer of shape `(seq_len, batch, embed_dim)` - attention_mask (`tf.Tensor`): attention mask of size - `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. - encoder_hidden_states (`tf.Tensor`): - cross attention input to the layer of shape `(seq_len, batch, embed_dim)` - encoder_attention_mask (`tf.Tensor`): encoder attention mask of size - `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. - layer_head_mask (`tf.Tensor`): mask for attention heads in a given layer of size - `(decoder_attention_heads,)` - cross_attn_layer_head_mask (`tf.Tensor`): mask for heads of the cross-attention module. - `(decoder_attention_heads,)` - past_key_value (`Tuple(tf.Tensor)`): cached past key and value projection states - """ - residual = hidden_states - - # Self Attention - # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 - self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None - # add present self-attn cache to positions 1,2 of present_key_value tuple - hidden_states, self_attn_weights, present_key_value = self.self_attn( - hidden_states=hidden_states, - past_key_value=self_attn_past_key_value, - attention_mask=attention_mask, - layer_head_mask=layer_head_mask, - ) - hidden_states = self.dropout(hidden_states, training=training) - hidden_states = residual + hidden_states - hidden_states = self.self_attn_layer_norm(hidden_states) - - # Cross-Attention Block - cross_attn_present_key_value = None - cross_attn_weights = None - if encoder_hidden_states is not None: - residual = hidden_states - - # cross_attn cached key/values tuple is at positions 3,4 of present_key_value tuple - cross_attn_past_key_value = past_key_value[-2:] if past_key_value is not None else None - hidden_states, cross_attn_weights, cross_attn_present_key_value = self.encoder_attn( - hidden_states=hidden_states, - key_value_states=encoder_hidden_states, - attention_mask=encoder_attention_mask, - layer_head_mask=cross_attn_layer_head_mask, - past_key_value=cross_attn_past_key_value, - ) - hidden_states = self.dropout(hidden_states, training=training) - hidden_states = residual + hidden_states - hidden_states = self.encoder_attn_layer_norm(hidden_states) - - # add cross-attn to positions 3,4 of present_key_value tuple - present_key_value = present_key_value + cross_attn_present_key_value - - # Fully Connected - residual = hidden_states - hidden_states = self.activation_fn(self.fc1(hidden_states)) - hidden_states = self.activation_dropout(hidden_states, training=training) - hidden_states = self.fc2(hidden_states) - hidden_states = self.dropout(hidden_states, training=training) - hidden_states = residual + hidden_states - hidden_states = self.final_layer_norm(hidden_states) - - return ( - hidden_states, - self_attn_weights, - cross_attn_weights, - present_key_value, - ) - - -OPT_START_DOCSTRING = r""" - This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the - library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads - etc.) - - This model is also a [tf.keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it - as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and - behavior. - - - - TF 2.0 models accepts two formats as inputs: - - - having all inputs as keyword arguments (like PyTorch models), or - - having all inputs as a list, tuple or dict in the first positional arguments. - - This second option is useful when using [`tf.keras.Model.fit`] method which currently requires having all the - tensors in the first argument of the model call function: `model(inputs)`. - - If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the - first positional argument : - - - a single Tensor with `input_ids` only and nothing else: `model(input_ids)` - - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: - `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` - - a dictionary with one or several input Tensors associated to the input names given in the docstring: - `model({"input_ids": input_ids, "token_type_ids": token_type_ids})` - - - - Args: - config ([`OPTConfig`]): Model configuration class with all the parameters of the model. - Initializing with a config file does not load the weights associated with the model, only the - configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights. -""" - - -@add_start_docstrings( - "The bare OPT Model outputting raw hidden-states without any specific head on top.", - OPT_START_DOCSTRING, -) -class TFOPTPretrainedModel(TFPreTrainedModel): - """ - TFOPT Pretrained Model that inheritates from transformers.TFPreTrainedModel - - Args: - config: OPTConfig - """ - - config_class = OPTConfig - base_model_prefix = "model" - - @property - def dummy_inputs(self): - pad_token = 1 - input_ids = tf.cast(tf.convert_to_tensor(DUMMY_INPUTS), tf.int32) - decoder_input_ids = tf.cast(tf.convert_to_tensor(DUMMY_INPUTS), tf.int32) - dummy_inputs = { - "decoder_input_ids": decoder_input_ids, - "attention_mask": tf.math.not_equal(input_ids, pad_token), - "input_ids": input_ids, - } - return dummy_inputs - - @tf.function( - input_signature=[ - { - "input_ids": tf.TensorSpec((None, None), tf.int32, name="input_ids"), - "attention_mask": tf.TensorSpec((None, None), tf.int32, name="attention_mask"), - "decoder_input_ids": tf.TensorSpec((None, None), tf.int32, name="decoder_input_ids"), - "decoder_attention_mask": tf.TensorSpec((None, None), tf.int32, name="decoder_attention_mask"), - } - ] - ) - def serving(self, inputs): - output = self.call(inputs) - - return self.serving_output(output) - - -OPT_GENERATION_EXAMPLE = r""" - Summarization example: - - ```python - >>> from transformers import OPTTokenizer, TFOPTForConditionalGeneration - - >>> model = TFOPTForConditionalGeneration.from_pretrained("facebook/opt-large") - >>> tokenizer = OPTTokenizer.from_pretrained("facebook/opt-large") - - >>> ARTICLE_TO_SUMMARIZE = "My friends are cool but they eat too many carbs." - >>> inputs = tokenizer([ARTICLE_TO_SUMMARIZE], max_length=1024, return_tensors="tf") - - >>> # Generate Summary - >>> summary_ids = model.generate(inputs["input_ids"], num_beams=4, max_length=5) - >>> print(tokenizer.batch_decode(summary_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)) - ``` - - Mask filling example: - - ```python - >>> from transformers import OPTTokenizer, TFOPTForConditionalGeneration - - >>> tokenizer = OPTTokenizer.from_pretrained("facebook/opt-large") - >>> TXT = "My friends are but they eat too many carbs." - - >>> model = TFOPTForConditionalGeneration.from_pretrained("facebook/opt-large") - >>> input_ids = tokenizer([TXT], return_tensors="tf")["input_ids"] - >>> logits = model(input_ids).logits - >>> probs = tf.nn.softmax(logits[0]) - >>> # probs[5] is associated with the mask token - ``` -""" - - -OPT_INPUTS_DOCSTRING = r""" - Args: - input_ids (`tf.Tensor` of shape `({0})`): - Indices of input sequence tokens in the vocabulary. - - Indices can be obtained using [`BertTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`tf.Tensor` of shape `({0})`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - decoder_input_ids (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*): - Indices of decoder input sequence tokens in the vocabulary. - - Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are decoder input IDs?](../glossary#decoder-input-ids) - - OPT uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` - is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). - - For translation and summarization training, `decoder_input_ids` should be provided. If no - `decoder_input_ids` is provided, the model will create this tensor by shifting the `input_ids` to the right - for denoising pre-training following the paper. - decoder_attention_mask (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*): - will be made by default and ignore pad tokens. It is not recommended to set this for most use cases. - head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - decoder_head_mask (`tf.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`tf.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - encoder_outputs (`tf.FloatTensor`, *optional*): - hidden states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. - of shape `(batch_size, sequence_length, hidden_size)` is a sequence of - past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers`) - contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that - don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all - `decoder_input_ids` of shape `(batch_size, sequence_length)`. - use_cache (`bool`, *optional*, defaults to `True`): - If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see - `past_key_values`). Set to `False` during training, `True` during generation - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned - tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the - config will be used instead. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for - more detail. This argument can be used only in eager mode, in graph mode the value in the config will be - used instead. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. This argument can be used in - eager mode, in graph mode the value will always be set to True. - training (`bool`, *optional*, defaults to `False`): - Whether or not to use the model in training mode (some modules like dropout modules have different - behaviors between training and evaluation). -""" - - -@keras_serializable -# Copied from transformers.models.bart.modeling_tf_bart.TFBartEncoder with Bart->OPT -class TFOPTEncoder(tf.keras.layers.Layer): - config_class = OPTConfig - """ - Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a - [`TFOPTEncoderLayer`]. - - Args: - config: OPTConfig - """ - - def __init__(self, config: OPTConfig, embed_tokens: Optional[TFSharedEmbeddings] = None, **kwargs): - super().__init__(**kwargs) - self.config = config - self.dropout = tf.keras.layers.Dropout(config.dropout) - self.layerdrop = config.encoder_layerdrop - self.padding_idx = config.pad_token_id - self.max_source_positions = config.max_position_embeddings - self.embed_scale = tf.math.sqrt(float(config.d_model)) if config.scale_embedding else 1.0 - - self.embed_tokens = embed_tokens - self.embed_positions = TFOPTLearnedPositionalEmbedding( - config.max_position_embeddings, - config.d_model, - name="embed_positions", - ) - self.layers = [TFOPTEncoderLayer(config, name=f"layers.{i}") for i in range(config.encoder_layers)] - self.layernorm_embedding = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="layernorm_embedding") - - def get_embed_tokens(self): - return self.embed_tokens - - def set_embed_tokens(self, embed_tokens): - self.embed_tokens = embed_tokens - - @unpack_inputs - def call( - self, - input_ids: Optional[TFModelInputType] = None, - inputs_embeds: Optional[Union[np.ndarray, tf.Tensor]] = None, - attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, - head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - training: Optional[bool] = False, - ) -> Union[TFBaseModelOutput, Tuple[tf.Tensor]]: - """ - Args: - input_ids (`tf.Tensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you - provide it. - - Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, `optional): - Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): - Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. - This is useful if you want more control over how to convert `input_ids` indices into associated vectors - than the model's internal embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - """ - - if input_ids is not None and inputs_embeds is not None: - raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") - elif input_ids is not None: - input_shape = shape_list(input_ids) - elif inputs_embeds is not None: - input_shape = shape_list(inputs_embeds)[:-1] - else: - raise ValueError("You have to specify either input_ids or inputs_embeds") - - if inputs_embeds is None: - inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale - - embed_pos = self.embed_positions(input_shape) - hidden_states = inputs_embeds + embed_pos - hidden_states = self.layernorm_embedding(hidden_states) - hidden_states = self.dropout(hidden_states, training=training) - - # check attention mask and invert - if attention_mask is not None: - # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] - attention_mask = _expand_mask(attention_mask) - else: - attention_mask = None - - encoder_states = () if output_hidden_states else None - all_attentions = () if output_attentions else None - - # check if head_mask has a correct number of layers specified if desired - # The tf.debugging asserts are not compliant with XLA then they - # have to be disabled in other modes than eager. - if head_mask is not None and tf.executing_eagerly(): - tf.debugging.assert_equal( - shape_list(head_mask)[0], - len(self.layers), - message=f"The head_mask should be specified for {len(self.layers)} layers, but it is for {shape_list(head_mask)[0]}.", - ) - - # encoder layers - for idx, encoder_layer in enumerate(self.layers): - - if output_hidden_states: - encoder_states = encoder_states + (hidden_states,) - # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description) - dropout_probability = random.uniform(0, 1) - if training and (dropout_probability < self.layerdrop): # skip the layer - continue - - hidden_states, attn = encoder_layer( - hidden_states, - attention_mask, - head_mask[idx] if head_mask is not None else None, - ) - - if output_attentions: - all_attentions += (attn,) - - if output_hidden_states: - encoder_states = encoder_states + (hidden_states,) - - if not return_dict: - return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None) - return TFBaseModelOutput( - last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions - ) - - -@keras_serializable -# Copied from transformers.models.bart.modeling_tf_bart.TFBartDecoder with Bart->OPT -class TFOPTDecoder(tf.keras.layers.Layer): - config_class = OPTConfig - """ - Transformer decoder consisting of *config.decoder_layers* layers. Each layer is a [`TFOPTDecoderLayer`] - - Args: - config: OPTConfig - embed_tokens: output embedding - """ - - def __init__(self, config: OPTConfig, embed_tokens: Optional[TFSharedEmbeddings] = None, **kwargs): - super().__init__(**kwargs) - self.config = config - self.padding_idx = config.pad_token_id - self.embed_tokens = embed_tokens - self.layerdrop = config.decoder_layerdrop - self.embed_positions = TFOPTLearnedPositionalEmbedding( - config.max_position_embeddings, - config.d_model, - name="embed_positions", - ) - self.embed_scale = tf.math.sqrt(float(config.d_model)) if config.scale_embedding else 1.0 - self.layers = [TFOPTDecoderLayer(config, name=f"layers.{i}") for i in range(config.decoder_layers)] - self.layernorm_embedding = tf.keras.layers.LayerNormalization(epsilon=1e-5, name="layernorm_embedding") - - self.dropout = tf.keras.layers.Dropout(config.dropout) - - def get_embed_tokens(self): - return self.embed_tokens - - def set_embed_tokens(self, embed_tokens): - self.embed_tokens = embed_tokens - - @unpack_inputs - def call( - self, - input_ids: Optional[TFModelInputType] = None, - inputs_embeds: Optional[Union[np.ndarray, tf.Tensor]] = None, - attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, - encoder_hidden_states: Optional[Union[np.ndarray, tf.Tensor]] = None, - encoder_attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, - head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, - cross_attn_head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, - past_key_values: Optional[Tuple[Tuple[Union[np.ndarray, tf.Tensor]]]] = None, - use_cache: Optional[bool] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - training: Optional[bool] = False, - ) -> Union[TFBaseModelOutputWithPastAndCrossAttentions, Tuple[tf.Tensor]]: - r""" - Args: - input_ids (`tf.Tensor` of shape `(batch_size, sequence_length)`): - Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you - provide it. - - Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. - - [What are input IDs?](../glossary#input-ids) - attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - encoder_hidden_states (`tf.Tensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention - of the decoder. - encoder_attention_mask (`tf.Tensor` of shape `(batch_size, encoder_sequence_length)`, *optional*): - Mask to avoid performing cross-attention on padding tokens indices of encoder input_ids. Mask values - selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - - [What are attention masks?](../glossary#attention-mask) - head_mask (`tf.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - cross_attn_head_mask (`tf.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): - Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - - - 1 indicates the head is **not masked**, - - 0 indicates the head is **masked**. - - past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers` with each tuple having 2 tuples each of which has 2 tensors of shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden-states of the attention blocks. Can be used to speed up - decoding. - - If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those - that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of - all ``decoder_input_ids``` of shape `(batch_size, sequence_length)`. inputs_embeds (`tf.Tensor` of - shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing - `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more - control over how to convert `input_ids` indices into associated vectors than the model's internal - embedding lookup matrix. - output_attentions (`bool`, *optional*): - Whether or not to return the attentions tensors of all attention layers. See `attentions` under - returned tensors for more detail. - output_hidden_states (`bool`, *optional*): - Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors - for more detail. - return_dict (`bool`, *optional*): - Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. - """ - - if input_ids is not None and inputs_embeds is not None: - raise ValueError("You cannot specify both decoder_input_ids and decoder_inputs_embeds at the same time") - elif input_ids is not None: - input_shape = shape_list(input_ids) - elif inputs_embeds is not None: - input_shape = shape_list(inputs_embeds)[:-1] - else: - raise ValueError("You have to specify either decoder_input_ids or decoder_inputs_embeds") - - past_key_values_length = shape_list(past_key_values[0][0])[2] if past_key_values is not None else 0 - - # embed positions - positions = self.embed_positions(input_shape, past_key_values_length) - - if inputs_embeds is None: - inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale - - hidden_states = inputs_embeds - - # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] - if input_shape[-1] > 1: - combined_attention_mask = _make_causal_mask(input_shape, past_key_values_length=past_key_values_length) - else: - combined_attention_mask = _expand_mask( - tf.ones((input_shape[0], input_shape[1] + past_key_values_length)), tgt_len=input_shape[-1] - ) - - if attention_mask is not None: - combined_attention_mask = combined_attention_mask + _expand_mask(attention_mask, tgt_len=input_shape[-1]) - - if encoder_hidden_states is not None and encoder_attention_mask is not None: - # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] - encoder_attention_mask = _expand_mask(encoder_attention_mask, tgt_len=input_shape[-1]) - - hidden_states = self.layernorm_embedding(hidden_states + positions) - hidden_states = self.dropout(hidden_states, training=training) - - # decoder layers - all_hidden_states = () if output_hidden_states else None - all_self_attns = () if output_attentions else None - all_cross_attns = () if (output_attentions and encoder_hidden_states is not None) else None - present_key_values = () if use_cache else None - - # check if head_mask and cross_attn_head_mask have a correct number of layers specified if desired - # The tf.debugging asserts are not compliant with XLA then they - # have to be disabled in other modes than eager. - for attn_mask_name, attn_mask in [("head_mask", head_mask), ("cross_attn_head_mask", cross_attn_head_mask)]: - if attn_mask is not None and tf.executing_eagerly(): - tf.debugging.assert_equal( - shape_list(attn_mask)[0], - len(self.layers), - message=f"The {attn_mask_name} should be specified for {len(self.layers)} layers, but it is for {shape_list(attn_mask)[0]}.", - ) - - for idx, decoder_layer in enumerate(self.layers): - # add LayerDrop (see https://arxiv.org/abs/1909.11556 for description) - if output_hidden_states: - all_hidden_states += (hidden_states,) - - dropout_probability = random.uniform(0, 1) - - if training and (dropout_probability < self.layerdrop): - continue - - past_key_value = past_key_values[idx] if past_key_values is not None else None - - hidden_states, layer_self_attn, layer_cross_attn, present_key_value = decoder_layer( - hidden_states, - attention_mask=combined_attention_mask, - encoder_hidden_states=encoder_hidden_states, - encoder_attention_mask=encoder_attention_mask, - layer_head_mask=head_mask[idx] if head_mask is not None else None, - cross_attn_layer_head_mask=cross_attn_head_mask[idx] if cross_attn_head_mask is not None else None, - past_key_value=past_key_value, - ) - - if use_cache: - present_key_values += (present_key_value,) - - if output_attentions: - all_self_attns += (layer_self_attn,) - - if encoder_hidden_states is not None: - all_cross_attns += (layer_cross_attn,) - - if output_hidden_states: - all_hidden_states += (hidden_states,) - - if not return_dict: - return hidden_states, present_key_values, all_hidden_states, all_self_attns, all_cross_attns - else: - return TFBaseModelOutputWithPastAndCrossAttentions( - last_hidden_state=hidden_states, - past_key_values=present_key_values, - hidden_states=all_hidden_states, - attentions=all_self_attns, - cross_attentions=all_cross_attns, - ) - - -@keras_serializable -# Copied from transformers.models.bart.modeling_tf_bart.TFBartMainLayer with Bart->OPT -class TFOPTMainLayer(tf.keras.layers.Layer): - config_class = OPTConfig - - def __init__(self, config: OPTConfig, load_weight_prefix=None, **kwargs): - super().__init__(**kwargs) - self.config = config - self.shared = TFSharedEmbeddings(config.vocab_size, config.d_model, config.pad_token_id, name="model.shared") - - # set tf scope correctly - if load_weight_prefix is None: - load_weight_prefix = "model.shared" - - with tf.compat.v1.variable_scope(load_weight_prefix) as shared_abs_scope_name: - pass - - # Wraps layer to avoid problems with weight restoring and ensuring we're in the correct TF scope. - embed_tokens = TFWrappedEmbeddings(self.shared, abs_scope_name=shared_abs_scope_name) - embed_tokens.vocab_size = self.shared.vocab_size - embed_tokens.hidden_size = self.shared.hidden_size - - self.encoder = TFOPTEncoder(config, embed_tokens, name="encoder") - self.decoder = TFOPTDecoder(config, embed_tokens, name="decoder") - - def get_input_embeddings(self): - return self.shared - - def set_input_embeddings(self, new_embeddings): - self.shared.weight = new_embeddings - self.shared.vocab_size = self.shared.weight.shape[0] - # retrieve correct absolute scope for embed token wrapper - with tf.compat.v1.variable_scope("model.shared") as shared_abs_scope_name: - pass - # Wraps layer to avoid problems with weight restoring and ensuring we're in the correct TF scope. - embed_tokens = TFWrappedEmbeddings(self.shared, abs_scope_name=shared_abs_scope_name) - self.encoder.set_embed_tokens(embed_tokens) - self.decoder.set_embed_tokens(embed_tokens) - - @unpack_inputs - def call( - self, - input_ids: Optional[TFModelInputType] = None, - attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, - decoder_input_ids: Optional[Union[np.ndarray, tf.Tensor]] = None, - decoder_attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, - head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, - decoder_head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, - cross_attn_head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, - encoder_outputs: Optional[Union[Tuple, TFBaseModelOutput]] = None, - past_key_values: Optional[Tuple[Tuple[Union[np.ndarray, tf.Tensor]]]] = None, - inputs_embeds: Optional[Union[np.ndarray, tf.Tensor]] = None, - decoder_inputs_embeds: Optional[Union[np.ndarray, tf.Tensor]] = None, - use_cache: Optional[bool] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - training: Optional[bool] = False, - **kwargs - ) -> Union[TFSeq2SeqModelOutput, Tuple[tf.Tensor]]: - - if decoder_input_ids is None and decoder_inputs_embeds is None: - use_cache = False - - output_hidden_states = ( - output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states - ) - - if decoder_input_ids is None and input_ids is not None: - decoder_input_ids = shift_tokens_right( - input_ids, self.config.pad_token_id, self.config.decoder_start_token_id - ) - - if encoder_outputs is None: - encoder_outputs = self.encoder( - input_ids=input_ids, - attention_mask=attention_mask, - head_mask=head_mask, - inputs_embeds=inputs_embeds, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - training=training, - ) - # If the user passed a tuple for encoder_outputs, we wrap it in a TFBaseModelOutput when return_dict=True - elif return_dict and not isinstance(encoder_outputs, TFBaseModelOutput): - encoder_outputs = TFBaseModelOutput( - last_hidden_state=encoder_outputs[0], - hidden_states=encoder_outputs[1] if len(encoder_outputs) > 1 else None, - attentions=encoder_outputs[2] if len(encoder_outputs) > 2 else None, - ) - # If the user passed a TFBaseModelOutput for encoder_outputs, we wrap it in a tuple when return_dict=False - elif not return_dict and not isinstance(encoder_outputs, tuple): - encoder_outputs = encoder_outputs.to_tuple() - - decoder_outputs = self.decoder( - decoder_input_ids, - attention_mask=decoder_attention_mask, - encoder_hidden_states=encoder_outputs[0], - encoder_attention_mask=attention_mask, - head_mask=decoder_head_mask, - cross_attn_head_mask=cross_attn_head_mask, - past_key_values=past_key_values, - inputs_embeds=decoder_inputs_embeds, - use_cache=use_cache, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - training=training, - ) - - if not return_dict: - return decoder_outputs + encoder_outputs - - return TFSeq2SeqModelOutput( - last_hidden_state=decoder_outputs.last_hidden_state, - past_key_values=decoder_outputs.past_key_values, - decoder_hidden_states=decoder_outputs.hidden_states, - decoder_attentions=decoder_outputs.attentions, - cross_attentions=decoder_outputs.cross_attentions, - encoder_last_hidden_state=encoder_outputs.last_hidden_state, - encoder_hidden_states=encoder_outputs.hidden_states, - encoder_attentions=encoder_outputs.attentions, - ) - - -@add_start_docstrings( - "The bare TF OPT Model outputting raw hidden-states without any specific head on top.", - OPT_START_DOCSTRING, -) -# Copied from transformers.models.bart.modeling_tf_bart.TFBartModel with BART->OPT,Bart->OPT -class TFOPTModel(TFOPTPretrainedModel): - - _requires_load_weight_prefix = True - - def __init__(self, config: OPTConfig, load_weight_prefix=None, *inputs, **kwargs): - super().__init__(config, *inputs, **kwargs) - - self.model = TFOPTMainLayer(config, load_weight_prefix=load_weight_prefix, name="model") - - def get_encoder(self): - return self.model.encoder - - def get_decoder(self): - return self.model.decoder - - @add_start_docstrings_to_model_forward(OPT_INPUTS_DOCSTRING.format("batch_size, sequence_length")) - @add_code_sample_docstrings( - processor_class=_TOKENIZER_FOR_DOC, - checkpoint=_CHECKPOINT_FOR_DOC, - output_type=TFSeq2SeqModelOutput, - config_class=_CONFIG_FOR_DOC, - ) - @unpack_inputs - def call( - self, - input_ids: Optional[TFModelInputType] = None, - attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, - decoder_input_ids: Optional[Union[np.ndarray, tf.Tensor]] = None, - decoder_attention_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, - head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, - decoder_head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, - cross_attn_head_mask: Optional[Union[np.ndarray, tf.Tensor]] = None, - encoder_outputs: Optional[Union[Tuple, TFBaseModelOutput]] = None, - past_key_values: Optional[Tuple[Tuple[Union[np.ndarray, tf.Tensor]]]] = None, - inputs_embeds: Optional[Union[np.ndarray, tf.Tensor]] = None, - decoder_inputs_embeds: Optional[Union[np.ndarray, tf.Tensor]] = None, - use_cache: Optional[bool] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - training: Optional[bool] = False, - **kwargs - ) -> Union[TFBaseModelOutput, Tuple[tf.Tensor]]: - - outputs = self.model( - input_ids=input_ids, - attention_mask=attention_mask, - decoder_input_ids=decoder_input_ids, - decoder_attention_mask=decoder_attention_mask, - head_mask=head_mask, - decoder_head_mask=decoder_head_mask, - cross_attn_head_mask=cross_attn_head_mask, - encoder_outputs=encoder_outputs, - past_key_values=past_key_values, - inputs_embeds=inputs_embeds, - decoder_inputs_embeds=decoder_inputs_embeds, - use_cache=use_cache, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - training=training, - ) - - return outputs - - def serving_output(self, output): - pkv = tf.tuple(output.past_key_values)[1] if self.config.use_cache else None - dec_hs = tf.convert_to_tensor(output.decoder_hidden_states) if self.config.output_hidden_states else None - dec_attns = tf.convert_to_tensor(output.decoder_attentions) if self.config.output_attentions else None - cross_attns = tf.convert_to_tensor(output.cross_attentions) if self.config.output_attentions else None - enc_hs = tf.convert_to_tensor(output.encoder_hidden_states) if self.config.output_hidden_states else None - enc_attns = tf.convert_to_tensor(output.encoder_attentions) if self.config.output_attentions else None - - return TFSeq2SeqModelOutput( - last_hidden_state=output.last_hidden_state, - past_key_values=pkv, - decoder_hidden_states=dec_hs, - decoder_attentions=dec_attns, - cross_attentions=cross_attns, - encoder_last_hidden_state=output.encoder_last_hidden_state, - encoder_hidden_states=enc_hs, - encoder_attentions=enc_attns, - ) diff --git a/src/transformers/utils/dummy_flax_objects.py b/src/transformers/utils/dummy_flax_objects.py index e125683f6f5a5..a6c6e7926da1b 100644 --- a/src/transformers/utils/dummy_flax_objects.py +++ b/src/transformers/utils/dummy_flax_objects.py @@ -795,34 +795,6 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["flax"]) -class FlaxOPTDecoderPreTrainedModel(metaclass=DummyObject): - _backends = ["flax"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["flax"]) - - -class FlaxOPTForCausalLM(metaclass=DummyObject): - _backends = ["flax"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["flax"]) - - -class FlaxOPTModel(metaclass=DummyObject): - _backends = ["flax"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["flax"]) - - -class FlaxOPTPreTrainedModel(metaclass=DummyObject): - _backends = ["flax"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["flax"]) - - class FlaxPegasusForConditionalGeneration(metaclass=DummyObject): _backends = ["flax"] diff --git a/src/transformers/utils/dummy_tf_objects.py b/src/transformers/utils/dummy_tf_objects.py index 5afa7082be874..e089a267a0247 100644 --- a/src/transformers/utils/dummy_tf_objects.py +++ b/src/transformers/utils/dummy_tf_objects.py @@ -1616,20 +1616,6 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["tf"]) -class TFOPTModel(metaclass=DummyObject): - _backends = ["tf"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["tf"]) - - -class TFOPTPretrainedModel(metaclass=DummyObject): - _backends = ["tf"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["tf"]) - - class TFPegasusForConditionalGeneration(metaclass=DummyObject): _backends = ["tf"] diff --git a/tests/models/opt/test_modeling_flax_opt.py b/tests/models/opt/test_modeling_flax_opt.py deleted file mode 100644 index 25925b87f0709..0000000000000 --- a/tests/models/opt/test_modeling_flax_opt.py +++ /dev/null @@ -1,451 +0,0 @@ -# Copyright 2021 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import unittest - -import numpy as np -import timeout_decorator # noqa - -from transformers import OPTConfig, is_flax_available -from transformers.testing_utils import require_flax, slow - -from ...generation.test_generation_flax_utils import FlaxGenerationTesterMixin -from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask - - -if is_flax_available(): - import os - - # The slow tests are often failing with OOM error on GPU - # This makes JAX allocate exactly what is needed on demand, and deallocate memory that is no longer needed - # but will be slower as stated here https://jax.readthedocs.io/en/latest/gpu_memory_allocation.html - os.environ["XLA_PYTHON_CLIENT_ALLOCATOR"] = "platform" - - import jax - import jax.numpy as jnp - from transformers.models.opt.modeling_flax_opt import FlaxOPTModel, shift_tokens_right - - -def prepare_opt_inputs_dict( - config, - input_ids, - decoder_input_ids=None, - attention_mask=None, - decoder_attention_mask=None, - head_mask=None, - decoder_head_mask=None, - cross_attn_head_mask=None, -): - if attention_mask is None: - attention_mask = np.where(input_ids != config.pad_token_id, 1, 0) - if decoder_attention_mask is None: - decoder_attention_mask = np.where(decoder_input_ids != config.pad_token_id, 1, 0) - if head_mask is None: - head_mask = np.ones((config.encoder_layers, config.encoder_attention_heads)) - if decoder_head_mask is None: - decoder_head_mask = np.ones((config.decoder_layers, config.decoder_attention_heads)) - if cross_attn_head_mask is None: - cross_attn_head_mask = np.ones((config.decoder_layers, config.decoder_attention_heads)) - return { - "input_ids": input_ids, - "decoder_input_ids": decoder_input_ids, - "attention_mask": attention_mask, - "decoder_attention_mask": attention_mask, - } - - -class FlaxOPTModelTester: - def __init__( - self, - parent, - batch_size=13, - seq_length=7, - is_training=True, - use_labels=False, - vocab_size=99, - hidden_size=16, - num_hidden_layers=2, - num_attention_heads=4, - intermediate_size=4, - hidden_act="gelu", - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, - max_position_embeddings=32, - eos_token_id=2, - pad_token_id=1, - bos_token_id=0, - initializer_range=0.02, - ): - self.parent = parent - self.batch_size = batch_size - self.seq_length = seq_length - self.is_training = is_training - self.use_labels = use_labels - self.vocab_size = vocab_size - self.hidden_size = hidden_size - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - self.intermediate_size = intermediate_size - self.hidden_act = hidden_act - self.hidden_dropout_prob = hidden_dropout_prob - self.attention_probs_dropout_prob = attention_probs_dropout_prob - self.max_position_embeddings = max_position_embeddings - self.eos_token_id = eos_token_id - self.pad_token_id = pad_token_id - self.bos_token_id = bos_token_id - self.initializer_range = initializer_range - - def prepare_config_and_inputs(self): - input_ids = np.clip(ids_tensor([self.batch_size, self.seq_length - 1], self.vocab_size), 3, self.vocab_size) - input_ids = np.concatenate((input_ids, 2 * np.ones((self.batch_size, 1), dtype=np.int64)), -1) - - decoder_input_ids = shift_tokens_right(input_ids, 1, 2) - - config = OPTConfig( - vocab_size=self.vocab_size, - d_model=self.hidden_size, - encoder_layers=self.num_hidden_layers, - decoder_layers=self.num_hidden_layers, - encoder_attention_heads=self.num_attention_heads, - decoder_attention_heads=self.num_attention_heads, - encoder_ffn_dim=self.intermediate_size, - decoder_ffn_dim=self.intermediate_size, - dropout=self.hidden_dropout_prob, - attention_dropout=self.attention_probs_dropout_prob, - max_position_embeddings=self.max_position_embeddings, - eos_token_id=self.eos_token_id, - bos_token_id=self.bos_token_id, - pad_token_id=self.pad_token_id, - initializer_range=self.initializer_range, - use_cache=False, - ) - inputs_dict = prepare_opt_inputs_dict(config, input_ids, decoder_input_ids) - return config, inputs_dict - - def prepare_config_and_inputs_for_common(self): - config, inputs_dict = self.prepare_config_and_inputs() - return config, inputs_dict - - def check_use_cache_forward(self, model_class_name, config, inputs_dict): - max_decoder_length = 20 - model = model_class_name(config) - - encoder_outputs = model.encode(inputs_dict["input_ids"]) - - decoder_input_ids, decoder_attention_mask = ( - inputs_dict["decoder_input_ids"], - inputs_dict["decoder_attention_mask"], - ) - - past_key_values = model.init_cache(decoder_input_ids.shape[0], max_decoder_length, encoder_outputs) - decoder_attention_mask = jnp.ones((decoder_input_ids.shape[0], max_decoder_length), dtype="i4") - - decoder_position_ids = jnp.broadcast_to( - jnp.arange(decoder_input_ids.shape[-1] - 1)[None, :], - (decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1), - ) - outputs_cache = model.decode( - decoder_input_ids[:, :-1], - encoder_outputs, - decoder_attention_mask=decoder_attention_mask, - past_key_values=past_key_values, - decoder_position_ids=decoder_position_ids, - ) - - decoder_position_ids = jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]], dtype="i4") - outputs_cache_next = model.decode( - decoder_input_ids[:, -1:], - encoder_outputs, - decoder_attention_mask=decoder_attention_mask, - past_key_values=outputs_cache.past_key_values, - decoder_position_ids=decoder_position_ids, - ) - - outputs = model.decode(decoder_input_ids, encoder_outputs) - - diff = np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5]))) - self.parent.assertTrue(diff < 1e-3, msg=f"Max diff is {diff}") - - def check_use_cache_forward_with_attn_mask(self, model_class_name, config, inputs_dict): - max_decoder_length = 20 - model = model_class_name(config) - - encoder_outputs = model.encode(inputs_dict["input_ids"]) - - decoder_input_ids, decoder_attention_mask = ( - inputs_dict["decoder_input_ids"], - inputs_dict["decoder_attention_mask"], - ) - - decoder_attention_mask_cache = jnp.concatenate( - [ - decoder_attention_mask, - jnp.zeros((decoder_attention_mask.shape[0], max_decoder_length - decoder_attention_mask.shape[1])), - ], - axis=-1, - ) - - past_key_values = model.init_cache(decoder_input_ids.shape[0], max_decoder_length, encoder_outputs) - decoder_position_ids = jnp.broadcast_to( - jnp.arange(decoder_input_ids.shape[-1] - 1)[None, :], - (decoder_input_ids.shape[0], decoder_input_ids.shape[-1] - 1), - ) - - outputs_cache = model.decode( - decoder_input_ids[:, :-1], - encoder_outputs, - decoder_attention_mask=decoder_attention_mask_cache, - past_key_values=past_key_values, - decoder_position_ids=decoder_position_ids, - ) - decoder_position_ids = jnp.array(decoder_input_ids.shape[0] * [[decoder_input_ids.shape[-1] - 1]], dtype="i4") - outputs_cache_next = model.decode( - decoder_input_ids[:, -1:], - encoder_outputs, - past_key_values=outputs_cache.past_key_values, - decoder_attention_mask=decoder_attention_mask_cache, - decoder_position_ids=decoder_position_ids, - ) - - outputs = model.decode(decoder_input_ids, encoder_outputs, decoder_attention_mask=decoder_attention_mask) - - diff = np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5]))) - self.parent.assertTrue(diff < 1e-3, msg=f"Max diff is {diff}") - - -@require_flax -class OPTHeadTests(unittest.TestCase): - vocab_size = 99 - - def _get_config_and_data(self): - input_ids = np.array( - [ - [71, 82, 18, 33, 46, 91, 2], - [68, 34, 26, 58, 30, 82, 2], - [5, 97, 17, 39, 94, 40, 2], - [76, 83, 94, 25, 70, 78, 2], - [87, 59, 41, 35, 48, 66, 2], - [55, 13, 16, 58, 5, 2, 1], # note padding - [64, 27, 31, 51, 12, 75, 2], - [52, 64, 86, 17, 83, 39, 2], - [48, 61, 9, 24, 71, 82, 2], - [26, 1, 60, 48, 22, 13, 2], - [21, 5, 62, 28, 14, 76, 2], - [45, 98, 37, 86, 59, 48, 2], - [70, 70, 50, 9, 28, 0, 2], - ], - dtype=np.int64, - ) - - batch_size = input_ids.shape[0] - config = OPTConfig( - vocab_size=self.vocab_size, - d_model=24, - encoder_layers=2, - decoder_layers=2, - encoder_attention_heads=2, - decoder_attention_heads=2, - encoder_ffn_dim=32, - decoder_ffn_dim=32, - max_position_embeddings=48, - eos_token_id=2, - pad_token_id=1, - bos_token_id=0, - ) - return config, input_ids, batch_size - - def test_shift_tokens_right(self): - input_ids = np.array([[71, 82, 18, 33, 2, 1, 1], [68, 34, 26, 58, 30, 82, 2]], dtype=np.int64) - shifted = shift_tokens_right(input_ids, 1, 2) - n_pad_before = np.equal(input_ids, 1).astype(np.float32).sum() - n_pad_after = np.equal(shifted, 1).astype(np.float32).sum() - self.assertEqual(shifted.shape, input_ids.shape) - self.assertEqual(n_pad_after, n_pad_before - 1) - self.assertTrue(np.equal(shifted[:, 0], 2).all()) - - -@require_flax -class FlaxOPTModelTest(FlaxModelTesterMixin, unittest.TestCase, FlaxGenerationTesterMixin): - is_encoder_decoder = True - all_model_classes = (FlaxOPTModel,) if is_flax_available() else () - all_generative_model_classes = () if is_flax_available() else () - - def setUp(self): - self.model_tester = FlaxOPTModelTester(self) - - def test_use_cache_forward(self): - config, inputs_dict = self.model_tester.prepare_config_and_inputs() - for model_class in self.all_model_classes: - self.model_tester.check_use_cache_forward(model_class, config, inputs_dict) - - def test_use_cache_forward_with_attn_mask(self): - config, inputs_dict = self.model_tester.prepare_config_and_inputs() - for model_class in self.all_model_classes: - self.model_tester.check_use_cache_forward_with_attn_mask(model_class, config, inputs_dict) - - def test_encode(self): - config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - - for model_class in self.all_model_classes: - with self.subTest(model_class.__name__): - prepared_inputs_dict = self._prepare_for_class(inputs_dict, model_class) - model = model_class(config) - - @jax.jit - def encode_jitted(input_ids, attention_mask=None, **kwargs): - return model.encode(input_ids=input_ids, attention_mask=attention_mask) - - with self.subTest("JIT Enabled"): - jitted_outputs = encode_jitted(**prepared_inputs_dict).to_tuple() - - with self.subTest("JIT Disabled"): - with jax.disable_jit(): - outputs = encode_jitted(**prepared_inputs_dict).to_tuple() - - self.assertEqual(len(outputs), len(jitted_outputs)) - for jitted_output, output in zip(jitted_outputs, outputs): - self.assertEqual(jitted_output.shape, output.shape) - - def test_decode(self): - config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - - for model_class in self.all_model_classes: - with self.subTest(model_class.__name__): - model = model_class(config) - encoder_outputs = model.encode(inputs_dict["input_ids"], inputs_dict["attention_mask"]) - - prepared_inputs_dict = { - "decoder_input_ids": inputs_dict["decoder_input_ids"], - "decoder_attention_mask": inputs_dict["decoder_attention_mask"], - "encoder_outputs": encoder_outputs, - } - - @jax.jit - def decode_jitted(decoder_input_ids, decoder_attention_mask, encoder_outputs): - return model.decode( - decoder_input_ids=decoder_input_ids, - decoder_attention_mask=decoder_attention_mask, - encoder_outputs=encoder_outputs, - ) - - with self.subTest("JIT Enabled"): - jitted_outputs = decode_jitted(**prepared_inputs_dict).to_tuple() - - with self.subTest("JIT Disabled"): - with jax.disable_jit(): - outputs = decode_jitted(**prepared_inputs_dict).to_tuple() - - self.assertEqual(len(outputs), len(jitted_outputs)) - for jitted_output, output in zip(jitted_outputs, outputs): - self.assertEqual(jitted_output.shape, output.shape) - - @slow - def test_model_from_pretrained(self): - for model_class_name in self.all_model_classes: - model = model_class_name.from_pretrained("", from_pt=True) - # FlaxOPTForSequenceClassification expects eos token in input_ids - input_ids = np.ones((1, 1)) * model.config.eos_token_id - outputs = model(input_ids) - self.assertIsNotNone(outputs) - - -class FlaxOPTStandaloneDecoderModelTester: - def __init__( - self, - parent, - batch_size=13, - seq_length=7, - is_training=True, - use_attention_mask=True, - use_labels=False, - vocab_size=99, - hidden_size=16, - num_hidden_layers=2, - num_attention_heads=4, - intermediate_size=4, - hidden_act="gelu", - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, - max_position_embeddings=32, - eos_token_id=2, - pad_token_id=1, - bos_token_id=0, - initializer_range=0.02, - ): - self.parent = parent - self.batch_size = batch_size - self.seq_length = seq_length - self.is_training = is_training - self.use_attention_mask = use_attention_mask - self.use_labels = use_labels - self.vocab_size = vocab_size - self.hidden_size = hidden_size - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - self.intermediate_size = intermediate_size - self.hidden_act = hidden_act - self.hidden_dropout_prob = hidden_dropout_prob - self.attention_probs_dropout_prob = attention_probs_dropout_prob - self.max_position_embeddings = max_position_embeddings - self.eos_token_id = eos_token_id - self.pad_token_id = pad_token_id - self.bos_token_id = bos_token_id - self.initializer_range = initializer_range - - def prepare_config_and_inputs(self): - input_ids = jnp.clip(ids_tensor([self.batch_size, self.seq_length], self.vocab_size), 3, self.vocab_size) - - attention_mask = None - if self.use_attention_mask: - attention_mask = random_attention_mask([self.batch_size, self.seq_length]) - - config = OPTConfig( - vocab_size=self.vocab_size, - d_model=self.hidden_size, - encoder_layers=self.num_hidden_layers, - decoder_layers=self.num_hidden_layers, - encoder_attention_heads=self.num_attention_heads, - decoder_attention_heads=self.num_attention_heads, - encoder_ffn_dim=self.intermediate_size, - decoder_ffn_dim=self.intermediate_size, - dropout=self.hidden_dropout_prob, - attention_dropout=self.attention_probs_dropout_prob, - max_position_embeddings=self.max_position_embeddings, - eos_token_id=self.eos_token_id, - bos_token_id=self.bos_token_id, - pad_token_id=self.pad_token_id, - initializer_range=self.initializer_range, - use_cache=False, - ) - - return config, input_ids, attention_mask - - def prepare_config_and_inputs_for_common(self): - config_and_inputs = self.prepare_config_and_inputs() - config, input_ids, attention_mask = config_and_inputs - inputs_dict = {"input_ids": input_ids, "attention_mask": attention_mask} - return config, inputs_dict - - def prepare_config_and_inputs_for_decoder(self): - config, input_ids, attention_mask = self.prepare_config_and_inputs() - - encoder_hidden_states = floats_tensor([self.batch_size, self.seq_length, self.hidden_size]) - encoder_attention_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) - - return ( - config, - input_ids, - attention_mask, - encoder_hidden_states, - encoder_attention_mask, - ) diff --git a/tests/models/opt/test_modeling_tf_opt.py b/tests/models/opt/test_modeling_tf_opt.py deleted file mode 100644 index 81945ebdfc972..0000000000000 --- a/tests/models/opt/test_modeling_tf_opt.py +++ /dev/null @@ -1,325 +0,0 @@ -# coding=utf-8 -# Copyright 2021 The HuggingFace Inc. team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest - -from transformers import OPTConfig, is_tf_available -from transformers.testing_utils import require_tf - -from ...test_configuration_common import ConfigTester -from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor -from ...utils.test_modeling_tf_core import TFCoreModelTesterMixin - - -if is_tf_available(): - import tensorflow as tf - - from transformers import TFOPTModel - - -@require_tf -class TFOPTModelTester: - config_cls = OPTConfig - config_updates = {} - hidden_act = "gelu" - - def __init__( - self, - parent, - batch_size=13, - seq_length=7, - is_training=True, - use_labels=False, - vocab_size=99, - hidden_size=32, - num_hidden_layers=5, - num_attention_heads=4, - intermediate_size=37, - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, - max_position_embeddings=20, - eos_token_id=2, - pad_token_id=1, - bos_token_id=0, - ): - self.parent = parent - self.batch_size = batch_size - self.seq_length = seq_length - self.is_training = is_training - self.use_labels = use_labels - self.vocab_size = vocab_size - self.hidden_size = hidden_size - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - self.intermediate_size = intermediate_size - - self.hidden_dropout_prob = hidden_dropout_prob - self.attention_probs_dropout_prob = attention_probs_dropout_prob - self.max_position_embeddings = max_position_embeddings - self.eos_token_id = eos_token_id - self.pad_token_id = pad_token_id - self.bos_token_id = bos_token_id - - def prepare_config_and_inputs_for_common(self): - input_ids = ids_tensor([self.batch_size, self.seq_length - 1], self.vocab_size) - eos_tensor = tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size), 1) - input_ids = tf.concat([input_ids, eos_tensor], axis=1) - - decoder_input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) - - config = self.config_cls( - vocab_size=self.vocab_size, - d_model=self.hidden_size, - encoder_layers=self.num_hidden_layers, - decoder_layers=self.num_hidden_layers, - encoder_attention_heads=self.num_attention_heads, - decoder_attention_heads=self.num_attention_heads, - encoder_ffn_dim=self.intermediate_size, - decoder_ffn_dim=self.intermediate_size, - dropout=self.hidden_dropout_prob, - attention_dropout=self.attention_probs_dropout_prob, - max_position_embeddings=self.max_position_embeddings, - eos_token_ids=[2], - bos_token_id=self.bos_token_id, - pad_token_id=self.pad_token_id, - decoder_start_token_id=self.pad_token_id, - **self.config_updates, - ) - inputs_dict = prepare_opt_inputs_dict(config, input_ids, decoder_input_ids) - return config, inputs_dict - - def check_decoder_model_past_large_inputs(self, config, inputs_dict): - model = TFOPTModel(config=config).get_decoder() - input_ids = inputs_dict["input_ids"] - - input_ids = input_ids[:1, :] - attention_mask = inputs_dict["attention_mask"][:1, :] - head_mask = inputs_dict["head_mask"] - self.batch_size = 1 - - # first forward pass - outputs = model(input_ids, attention_mask=attention_mask, head_mask=head_mask, use_cache=True) - - output, past_key_values = outputs.to_tuple() - - # create hypothetical next token and extent to next_input_ids - next_tokens = ids_tensor((self.batch_size, 3), config.vocab_size) - next_attn_mask = tf.cast(ids_tensor((self.batch_size, 3), 2), tf.int8) - - # append to next input_ids and - next_input_ids = tf.concat([input_ids, next_tokens], axis=-1) - next_attention_mask = tf.concat([attention_mask, next_attn_mask], axis=-1) - - output_from_no_past = model(next_input_ids, attention_mask=next_attention_mask)[0] - output_from_past = model(next_tokens, attention_mask=next_attention_mask, past_key_values=past_key_values)[0] - - self.parent.assertEqual(next_tokens.shape[1], output_from_past.shape[1]) - - # select random slice - random_slice_idx = int(ids_tensor((1,), output_from_past.shape[-1])) - output_from_no_past_slice = output_from_no_past[:, -3:, random_slice_idx] - output_from_past_slice = output_from_past[:, :, random_slice_idx] - - # test that outputs are equal for slice - tf.debugging.assert_near(output_from_past_slice, output_from_no_past_slice, rtol=1e-3) - - -def prepare_opt_inputs_dict( - config, - input_ids, - decoder_input_ids, - attention_mask=None, - decoder_attention_mask=None, - head_mask=None, - decoder_head_mask=None, - cross_attn_head_mask=None, -): - if attention_mask is None: - attention_mask = tf.cast(tf.math.not_equal(input_ids, config.pad_token_id), tf.int8) - if decoder_attention_mask is None: - decoder_attention_mask = tf.concat( - [ - tf.ones(decoder_input_ids[:, :1].shape, dtype=tf.int8), - tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:], config.pad_token_id), tf.int8), - ], - axis=-1, - ) - if head_mask is None: - head_mask = tf.ones((config.encoder_layers, config.encoder_attention_heads)) - if decoder_head_mask is None: - decoder_head_mask = tf.ones((config.decoder_layers, config.decoder_attention_heads)) - if cross_attn_head_mask is None: - cross_attn_head_mask = tf.ones((config.decoder_layers, config.decoder_attention_heads)) - return { - "input_ids": input_ids, - "decoder_input_ids": decoder_input_ids, - "attention_mask": attention_mask, - "decoder_attention_mask": decoder_attention_mask, - "head_mask": head_mask, - "decoder_head_mask": decoder_head_mask, - "cross_attn_head_mask": cross_attn_head_mask, - } - - -@require_tf -class TFOPTModelTest(TFModelTesterMixin, TFCoreModelTesterMixin, unittest.TestCase): - all_model_classes = (TFOPTModel) if is_tf_available() else () - all_generative_model_classes = () if is_tf_available() else () - is_encoder_decoder = True - test_pruning = False - test_onnx = True - onnx_min_opset = 10 - - def setUp(self): - self.model_tester = TFOPTModelTester(self) - self.config_tester = ConfigTester(self, config_class=OPTConfig) - - def test_config(self): - self.config_tester.run_common_tests() - - def test_decoder_model_past_large_inputs(self): - config_and_inputs = self.model_tester.prepare_config_and_inputs_for_common() - self.model_tester.check_decoder_model_past_large_inputs(*config_and_inputs) - - def test_model_common_attributes(self): - config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - - for model_class in self.all_model_classes: - model = model_class(config) - assert isinstance(model.get_input_embeddings(), tf.keras.layers.Layer) - - if model_class in self.all_generative_model_classes: - x = model.get_output_embeddings() - assert isinstance(x, tf.keras.layers.Layer) - name = model.get_bias() - assert isinstance(name, dict) - for k, v in name.items(): - assert isinstance(v, tf.Variable) - else: - x = model.get_output_embeddings() - assert x is None - name = model.get_bias() - assert name is None - - def test_resize_token_embeddings(self): - config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - - def _get_word_embedding_weight(model, embedding_layer): - if hasattr(embedding_layer, "weight"): - return embedding_layer.weight - else: - # Here we build the word embeddings weights if not exists. - # And then we retry to get the attribute once built. - model(model.dummy_inputs) - if hasattr(embedding_layer, "weight"): - return embedding_layer.weight - else: - return None - - for model_class in self.all_model_classes: - for size in [config.vocab_size - 10, config.vocab_size + 10, None]: - # build the embeddings - model = model_class(config=config) - old_input_embeddings = _get_word_embedding_weight(model, model.get_input_embeddings()) - old_output_embeddings = _get_word_embedding_weight(model, model.get_output_embeddings()) - old_final_logits_bias = model.get_bias() - - # reshape the embeddings - model.resize_token_embeddings(size) - new_input_embeddings = _get_word_embedding_weight(model, model.get_input_embeddings()) - new_output_embeddings = _get_word_embedding_weight(model, model.get_output_embeddings()) - new_final_logits_bias = model.get_bias() - - # check that the resized embeddings size matches the desired size. - assert_size = size if size is not None else config.vocab_size - - self.assertEqual(new_input_embeddings.shape[0], assert_size) - - # check that weights remain the same after resizing - models_equal = True - for p1, p2 in zip(old_input_embeddings.value(), new_input_embeddings.value()): - if tf.math.reduce_sum(tf.math.abs(p1 - p2)) > 0: - models_equal = False - self.assertTrue(models_equal) - - if old_output_embeddings is not None and new_output_embeddings is not None: - self.assertEqual(new_output_embeddings.shape[0], assert_size) - - models_equal = True - for p1, p2 in zip(old_output_embeddings.value(), new_output_embeddings.value()): - if tf.math.reduce_sum(tf.math.abs(p1 - p2)) > 0: - models_equal = False - self.assertTrue(models_equal) - - if old_final_logits_bias is not None and new_final_logits_bias is not None: - old_final_logits_bias = old_final_logits_bias["final_logits_bias"] - new_final_logits_bias = new_final_logits_bias["final_logits_bias"] - self.assertEqual(new_final_logits_bias.shape[0], 1) - self.assertEqual(new_final_logits_bias.shape[1], assert_size) - - models_equal = True - for old, new in zip(old_final_logits_bias.value(), new_final_logits_bias.value()): - for p1, p2 in zip(old, new): - if tf.math.reduce_sum(tf.math.abs(p1 - p2)) > 0: - models_equal = False - self.assertTrue(models_equal) - - def test_saved_model_creation(self): - # This test is too long (>30sec) and makes fail the CI - pass - - -def _assert_tensors_equal(a, b, atol=1e-12, prefix=""): - """If tensors not close, or a and b arent both tensors, raise a nice Assertion error.""" - if a is None and b is None: - return True - try: - if tf.debugging.assert_near(a, b, atol=atol): - return True - raise - except Exception: - if len(prefix) > 0: - prefix = f"{prefix}: " - raise AssertionError(f"{prefix}{a} != {b}") - - -def _long_tensor(tok_lst): - return tf.constant(tok_lst, dtype=tf.int32) - - -@require_tf -class TFOPTHeadTests(unittest.TestCase): - vocab_size = 99 - - def _get_config_and_data(self): - eos_column_vector = tf.ones((4, 1), dtype=tf.int32) * 2 - input_ids = tf.concat([ids_tensor((4, 6), self.vocab_size - 3) + 3, eos_column_vector], axis=1) - batch_size = input_ids.shape[0] - config = OPTConfig( - vocab_size=self.vocab_size, - d_model=24, - encoder_layers=2, - decoder_layers=2, - encoder_attention_heads=2, - decoder_attention_heads=2, - encoder_ffn_dim=32, - decoder_ffn_dim=32, - max_position_embeddings=48, - eos_token_id=2, - pad_token_id=1, - bos_token_id=0, - decoder_start_token_id=2, - ) - return config, input_ids, batch_size From d6338329b122bbb7c2dcca8994a5519563041610 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Wed, 11 May 2022 11:53:55 +0200 Subject: [PATCH 083/145] clean traces of Flax and Tf --- docs/source/en/model_doc/opt.mdx | 23 ------------------- src/transformers/__init__.py | 11 --------- .../models/auto/modeling_flax_auto.py | 2 -- .../models/auto/modeling_tf_auto.py | 1 - src/transformers/models/opt/__init__.py | 21 ----------------- .../models/opt/configuration_opt.py | 9 +------- utils/check_repo.py | 1 - 7 files changed, 1 insertion(+), 67 deletions(-) diff --git a/docs/source/en/model_doc/opt.mdx b/docs/source/en/model_doc/opt.mdx index 5c117193e987f..6cda45da86045 100644 --- a/docs/source/en/model_doc/opt.mdx +++ b/docs/source/en/model_doc/opt.mdx @@ -51,26 +51,3 @@ The original code can be found [here](). [[autodoc]] OPTForCausalLM - forward -## TFOPTModel - -[[autodoc]] TFOPTModel - - call - -## TFOPTPretrainedModel - -[[autodoc]] TFOPTPretrainedModel - - call - - -## FlaxOPTModel - -[[autodoc]] FlaxOPTModel - - __call__ - - encode - - decode - - -## FlaxOPTForCausalLM - -[[autodoc]] FlaxOPTForCausalLM - - __call__ diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 32b421eb0c9d2..8f887f7542326 100755 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -2125,7 +2125,6 @@ "TFOpenAIGPTPreTrainedModel", ] ) - _import_structure["models.opt"].extend(["TFOPTModel", "TFOPTPretrainedModel"]) _import_structure["models.pegasus"].extend( ["TFPegasusForConditionalGeneration", "TFPegasusModel", "TFPegasusPreTrainedModel"] ) @@ -2464,14 +2463,6 @@ ] ) _import_structure["models.mt5"].extend(["FlaxMT5ForConditionalGeneration", "FlaxMT5Model"]) - _import_structure["models.opt"].extend( - [ - "FlaxOPTDecoderPreTrainedModel", - "FlaxOPTForCausalLM", - "FlaxOPTModel", - "FlaxOPTPreTrainedModel", - ] - ) _import_structure["models.pegasus"].extend( [ "FlaxPegasusForConditionalGeneration", @@ -4286,7 +4277,6 @@ TFOpenAIGPTModel, TFOpenAIGPTPreTrainedModel, ) - from .models.opt import TFOPTModel, TFOPTPretrainedModel from .models.pegasus import TFPegasusForConditionalGeneration, TFPegasusModel, TFPegasusPreTrainedModel from .models.rag import TFRagModel, TFRagPreTrainedModel, TFRagSequenceForGeneration, TFRagTokenForGeneration from .models.rembert import ( @@ -4549,7 +4539,6 @@ FlaxMBartPreTrainedModel, ) from .models.mt5 import FlaxMT5ForConditionalGeneration, FlaxMT5Model - from .models.opt import FlaxOPTDecoderPreTrainedModel, FlaxOPTForCausalLM, FlaxOPTModel, FlaxOPTPreTrainedModel from .models.pegasus import FlaxPegasusForConditionalGeneration, FlaxPegasusModel, FlaxPegasusPreTrainedModel from .models.roberta import ( FlaxRobertaForCausalLM, diff --git a/src/transformers/models/auto/modeling_flax_auto.py b/src/transformers/models/auto/modeling_flax_auto.py index 219d3b861d661..78803178bec0c 100644 --- a/src/transformers/models/auto/modeling_flax_auto.py +++ b/src/transformers/models/auto/modeling_flax_auto.py @@ -40,7 +40,6 @@ ("beit", "FlaxBeitModel"), ("big_bird", "FlaxBigBirdModel"), ("bart", "FlaxBartModel"), - ("opt", "FlaxOPTModel"), ("gpt2", "FlaxGPT2Model"), ("gpt_neo", "FlaxGPTNeoModel"), ("gptj", "FlaxGPTJModel"), @@ -128,7 +127,6 @@ ("gptj", "FlaxGPTJForCausalLM"), ("xglm", "FlaxXGLMForCausalLM"), ("bart", "FlaxBartForCausalLM"), - ("opt", "FlaxOPTForCausalLM"), ("bert", "FlaxBertForCausalLM"), ("roberta", "FlaxRobertaForCausalLM"), ("big_bird", "FlaxBigBirdForCausalLM"), diff --git a/src/transformers/models/auto/modeling_tf_auto.py b/src/transformers/models/auto/modeling_tf_auto.py index 87dc0219ed790..e2c1f3017453c 100644 --- a/src/transformers/models/auto/modeling_tf_auto.py +++ b/src/transformers/models/auto/modeling_tf_auto.py @@ -45,7 +45,6 @@ ("distilbert", "TFDistilBertModel"), ("albert", "TFAlbertModel"), ("bart", "TFBartModel"), - ("opt", "TFOPTModel"), ("camembert", "TFCamembertModel"), ("xlm-roberta", "TFXLMRobertaModel"), ("longformer", "TFLongformerModel"), diff --git a/src/transformers/models/opt/__init__.py b/src/transformers/models/opt/__init__.py index 7192f1255d079..19db2c69e4b10 100644 --- a/src/transformers/models/opt/__init__.py +++ b/src/transformers/models/opt/__init__.py @@ -33,16 +33,6 @@ "OPTPretrainedModel", ] -if is_tf_available(): - _import_structure["modeling_tf_opt"] = ["TFOPTModel", "TFOPTPretrainedModel"] - -if is_flax_available(): - _import_structure["modeling_flax_opt"] = [ - "FlaxOPTDecoderPreTrainedModel", - "FlaxOPTForCausalLM", - "FlaxOPTModel", - "FlaxOPTPreTrainedModel", - ] if TYPE_CHECKING: from .configuration_opt import OPT_PRETRAINED_CONFIG_ARCHIVE_MAP, OPTConfig, OPTOnnxConfig @@ -50,17 +40,6 @@ if is_torch_available(): from .modeling_opt import OPT_PRETRAINED_MODEL_ARCHIVE_LIST, OPTForCausalLM, OPTModel, OPTPretrainedModel - if is_tf_available(): - from .modeling_tf_opt import TFOPTModel, TFOPTPretrainedModel - - if is_flax_available(): - from .modeling_flax_opt import ( - FlaxOPTDecoderPreTrainedModel, - FlaxOPTForCausalLM, - FlaxOPTModel, - FlaxOPTPreTrainedModel, - ) - else: import sys diff --git a/src/transformers/models/opt/configuration_opt.py b/src/transformers/models/opt/configuration_opt.py index 6f68e1026008c..13a84180639c2 100644 --- a/src/transformers/models/opt/configuration_opt.py +++ b/src/transformers/models/opt/configuration_opt.py @@ -40,7 +40,7 @@ class OPTConfig(PretrainedConfig): Args: vocab_size (`int`, *optional*, defaults to 50272): Vocabulary size of the OPT model. Defines the number of different tokens that can be represented by the - `inputs_ids` passed when calling [`OPTModel`] or [`TFOPTModel`]. + `inputs_ids` passed when calling [`OPTModel`] d_model (`int`, *optional*, defaults to 768): Dimensionality of the layers and the pooler layer. num_hidden_layers (`int`, *optional*, defaults to 12): @@ -141,10 +141,3 @@ def __init__( **kwargs, ) - # ensure backward compatibility for OPT CNN models - if self.forced_bos_token_id is None and kwargs.get("force_bos_token_to_be_generated", False): - self.forced_bos_token_id = self.bos_token_id - warnings.warn( - f"Please make sure the config includes `forced_bos_token_id={self.bos_token_id}` in future versions. " - "The config can simply be saved and uploaded again to be fixed." - ) diff --git a/utils/check_repo.py b/utils/check_repo.py index ecee201f0bce2..a1bdae71929c4 100644 --- a/utils/check_repo.py +++ b/utils/check_repo.py @@ -92,7 +92,6 @@ "SeparableConv1D", # Building part of bigger (tested) model. "FlaxBartForCausalLM", # Building part of bigger (tested) model. "FlaxBertForCausalLM", # Building part of bigger (tested) model. Tested implicitly through FlaxRobertaForCausalLM. - "FlaxOPTForCausalLM", # "OPTDecoderWrapper", ] From 85ce8e87baa56ccae827208b995b773c62f41193 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Wed, 11 May 2022 11:55:18 +0200 Subject: [PATCH 084/145] fix mask --- src/transformers/models/opt/modeling_opt.py | 26 ++++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index eda19a47e2a43..50852d590a90b 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -81,7 +81,7 @@ def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] return inverted_mask.masked_fill(inverted_mask.bool(), torch.finfo(dtype).min) -def make_positions(attention_mask, padding_idx: int): +def make_positions(mask, padding_idx: int): """Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols are ignored. @@ -90,8 +90,8 @@ def make_positions(attention_mask, padding_idx: int): # balanced to both work with ONNX export and XLA. In particular XLA # prefers ints, cumsum defaults to output longs, and ONNX doesn't know # how to handle the dtype kwarg in cumsum. - mask = attention_mask - return (torch.cumsum(mask, dim=1).type_as(mask) * mask).long() + padding_idx + positions = (torch.cumsum(mask, dim=1).type_as(mask) * mask).long() + padding_idx + return positions class OPTLearnedPositionalEmbedding(nn.Embedding): @@ -122,6 +122,7 @@ def forward( # padding. if positions is None: + attention_mask = attention_mask.long() positions = make_positions(attention_mask, self.padding_idx) return F.embedding( @@ -505,6 +506,7 @@ def __init__(self, config: OPTConfig): if self.padding_idx is not None: num_embeddings = config.max_position_embeddings + 2 + self.embed_positions = OPTLearnedPositionalEmbedding( num_embeddings, config.d_model, @@ -633,6 +635,7 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale + print("hf 2", inputs_embeds.abs().sum()) # embed positions attention_mask = attention_mask if attention_mask is not None else torch.ones(inputs_embeds.shape[:2], dtype=torch.bool, device=inputs_embeds.device) @@ -646,6 +649,7 @@ def forward( inputs_embeds = self.project_in(inputs_embeds) hidden_states = inputs_embeds + positions + print("hid", hidden_states.abs().sum()) hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) @@ -705,12 +709,8 @@ def custom_forward(*inputs): use_cache=use_cache, ) - if self.layer_norm: - new_layer_output = (self.layer_norm(layer_outputs[0]),) - new_layer_output += (layer_outputs[1:],) - layer_outputs = new_layer_output - hidden_states = layer_outputs[0] + print("hf attn", hidden_states.abs().sum()) if use_cache: next_decoder_cache += (layer_outputs[3 if output_attentions else 1],) @@ -718,9 +718,16 @@ def custom_forward(*inputs): if output_attentions: all_self_attns += (layer_outputs[1],) + if self.layer_norm: + hidden_states = self.layer_norm(hidden_states) + + print("hf final 1", hidden_states.abs().sum()) + if self.project_out is not None: hidden_states = self.project_out(hidden_states) + print("hf final 2", hidden_states.abs().sum()) + # add hidden states from the last decoder layer if output_hidden_states: all_hidden_states += (hidden_states,) @@ -951,9 +958,10 @@ def forward( return_dict=return_dict, ) - # logits = self.lm_head(outputs[0]).permute(1, 0, 2).contiguous() logits = self.lm_head(outputs[0]).contiguous() + print("hf final 3", logits.abs().sum()) + loss = None if labels is not None: loss_fct = CrossEntropyLoss() From 95d7ead7c1ad8fa4e87322b00e72ad6f42823f88 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Wed, 11 May 2022 12:33:26 +0200 Subject: [PATCH 085/145] fixed positionnal embedding length when past key value is provoded --- src/transformers/models/opt/modeling_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 50852d590a90b..c34ce7909f0a5 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -639,7 +639,7 @@ def forward( # embed positions attention_mask = attention_mask if attention_mask is not None else torch.ones(inputs_embeds.shape[:2], dtype=torch.bool, device=inputs_embeds.device) - positions = self.embed_positions(attention_mask) + positions = self.embed_positions(attention_mask)[:,past_key_values_length:,:] attention_mask = self._prepare_decoder_attention_mask( attention_mask, input_shape, inputs_embeds, past_key_values_length From 412bdab76dedc7301a798c0d2e73d7239147c2a6 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Wed, 11 May 2022 13:00:54 +0200 Subject: [PATCH 086/145] get 125m, 6.7b to work --- .../models/opt/configuration_opt.py | 7 ++--- ..._original_pytorch_checkpoint_to_pytorch.py | 29 ++++++------------- src/transformers/models/opt/modeling_opt.py | 29 ++++++++++--------- 3 files changed, 26 insertions(+), 39 deletions(-) diff --git a/src/transformers/models/opt/configuration_opt.py b/src/transformers/models/opt/configuration_opt.py index 13a84180639c2..4bd25e91e64c8 100644 --- a/src/transformers/models/opt/configuration_opt.py +++ b/src/transformers/models/opt/configuration_opt.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. """ OPT model configuration""" -import warnings - from ...configuration_utils import PretrainedConfig from ...utils import logging @@ -40,7 +38,7 @@ class OPTConfig(PretrainedConfig): Args: vocab_size (`int`, *optional*, defaults to 50272): Vocabulary size of the OPT model. Defines the number of different tokens that can be represented by the - `inputs_ids` passed when calling [`OPTModel`] + `inputs_ids` passed when calling [`OPTModel`] d_model (`int`, *optional*, defaults to 768): Dimensionality of the layers and the pooler layer. num_hidden_layers (`int`, *optional*, defaults to 12): @@ -111,7 +109,7 @@ def __init__( decoder_start_token_id=2, forced_eos_token_id=2, output_projection=True, - decoder_layernorm=True, + decoder_layernorm=False, # think we can delete this - to check **kwargs ): self.vocab_size = vocab_size @@ -140,4 +138,3 @@ def __init__( forced_eos_token_id=forced_eos_token_id, **kwargs, ) - diff --git a/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py index c42aa7281aadc..60ef10af2931a 100644 --- a/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py @@ -28,23 +28,6 @@ logger = logging.get_logger(__name__) -def remove_ignore_keys_(state_dict): - ignore_keys = [ - "encoder.version", - "decoder.version", - "model.encoder.version", - "model.decoder.version", - "_float_tensor", - ] - for k in ignore_keys: - state_dict.pop(k, None) - - -def rename_key(dct, old, new): - val = dct.pop(old) - dct[new] = val - - def load_checkpoint(checkpoint_path): """Checkpoint path should end in model.pt""" sd = torch.load(checkpoint_path, map_location="cpu") @@ -52,8 +35,11 @@ def load_checkpoint(checkpoint_path): sd = torch.load(checkpoint_path, map_location="cpu")["model"] # pop unnecessary weights - if "decoder.version" in sd: - sd.pop("decoder.version") + keys_to_delete = ["decoder.version", "decoder.layer_norm.weight", "decoder.layer_norm.bias"] + for key in keys_to_delete: + if key in sd: + sd.pop(key) + return sd @@ -64,7 +50,10 @@ def convert_opt_checkpoint(checkpoint_path, pytorch_dump_folder_path, config=Non """ state_dict = load_checkpoint(checkpoint_path) - config = OPTConfig() + if config is not None: + config = OPTConfig.from_pretrained(config) + else: + config = OPTConfig() model = OPTModel(config).half().eval() model.load_state_dict(state_dict) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 50852d590a90b..f3ce3555b638b 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -326,6 +326,10 @@ def forward( residual = hidden_states # Self Attention + # TODO(ArthurZ) could you add a flag to the config "do_layer_norm_before = True/False" to + # differentiate between 350m and all other checkpoitns + # Before + hidden_states = self.self_attn_layer_norm(hidden_states) hidden_states, self_attn_weights, present_key_value = self.self_attn( hidden_states=hidden_states, past_key_value=past_key_value, @@ -335,12 +339,15 @@ def forward( ) hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) hidden_states = residual + hidden_states - hidden_states = self.self_attn_layer_norm(hidden_states) + # After +# hidden_states = self.self_attn_layer_norm(hidden_states) # Fully Connected hidden_states_shape = hidden_states.shape hidden_states = hidden_states.reshape(-1, hidden_states.size(-1)) residual = hidden_states + # Before + hidden_states = self.final_layer_norm(hidden_states) hidden_states = self.fc1(hidden_states) hidden_states = self.activation_fn(hidden_states) @@ -348,7 +355,8 @@ def forward( hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) hidden_states = (residual + hidden_states).view(hidden_states_shape) - hidden_states = self.final_layer_norm(hidden_states) + # After +# hidden_states = self.final_layer_norm(hidden_states) outputs = (hidden_states,) @@ -523,7 +531,9 @@ def __init__(self, config: OPTConfig): else: self.project_in = None - self.layer_norm = nn.LayerNorm(config.d_model) if config.decoder_layernorm else None + # Think we can delete this +# self.layer_norm = nn.LayerNorm(config.d_model) if config.decoder_layernorm else None + self.layer_norm = None self.layers = nn.ModuleList([OPTDecoderLayer(config) for _ in range(config.num_hidden_layers)]) self.gradient_checkpointing = False @@ -635,7 +645,6 @@ def forward( if inputs_embeds is None: inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale - print("hf 2", inputs_embeds.abs().sum()) # embed positions attention_mask = attention_mask if attention_mask is not None else torch.ones(inputs_embeds.shape[:2], dtype=torch.bool, device=inputs_embeds.device) @@ -649,7 +658,6 @@ def forward( inputs_embeds = self.project_in(inputs_embeds) hidden_states = inputs_embeds + positions - print("hid", hidden_states.abs().sum()) hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) @@ -710,7 +718,6 @@ def custom_forward(*inputs): ) hidden_states = layer_outputs[0] - print("hf attn", hidden_states.abs().sum()) if use_cache: next_decoder_cache += (layer_outputs[3 if output_attentions else 1],) @@ -718,16 +725,12 @@ def custom_forward(*inputs): if output_attentions: all_self_attns += (layer_outputs[1],) - if self.layer_norm: - hidden_states = self.layer_norm(hidden_states) - - print("hf final 1", hidden_states.abs().sum()) +# if self.layer_norm: +# hidden_states = self.layer_norm(hidden_states) if self.project_out is not None: hidden_states = self.project_out(hidden_states) - print("hf final 2", hidden_states.abs().sum()) - # add hidden states from the last decoder layer if output_hidden_states: all_hidden_states += (hidden_states,) @@ -960,8 +963,6 @@ def forward( logits = self.lm_head(outputs[0]).contiguous() - print("hf final 3", logits.abs().sum()) - loss = None if labels is not None: loss_fct = CrossEntropyLoss() From cc1b4c9c8185943261146fbd3157bc998b3f1ff8 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Wed, 11 May 2022 13:07:25 +0200 Subject: [PATCH 087/145] Added do_layer_norm --- src/transformers/models/opt/configuration_opt.py | 4 ++-- src/transformers/models/opt/modeling_opt.py | 13 +++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/transformers/models/opt/configuration_opt.py b/src/transformers/models/opt/configuration_opt.py index 4bd25e91e64c8..c0af2e1c1a0df 100644 --- a/src/transformers/models/opt/configuration_opt.py +++ b/src/transformers/models/opt/configuration_opt.py @@ -109,7 +109,7 @@ def __init__( decoder_start_token_id=2, forced_eos_token_id=2, output_projection=True, - decoder_layernorm=False, # think we can delete this - to check + do_layer_norm_before = True, **kwargs ): self.vocab_size = vocab_size @@ -129,7 +129,7 @@ def __init__( self.use_cache = use_cache self.scale_embedding = scale_embedding # scale factor will be sqrt(d_model) if True self.output_projection = output_projection - self.decoder_layernorm = decoder_layernorm + self.do_layer_norm_before = do_layer_norm_before super().__init__( pad_token_id=pad_token_id, bos_token_id=bos_token_id, diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 31d58161063ac..c7fd45f62317e 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -291,6 +291,7 @@ def __init__(self, config: OPTConfig): dropout=config.attention_dropout, is_decoder=True, ) + self.do_layer_norm_before = config.do_layer_norm_before self.dropout = config.dropout self.activation_fn = ACT2FN[config.activation_function] @@ -329,7 +330,8 @@ def forward( # TODO(ArthurZ) could you add a flag to the config "do_layer_norm_before = True/False" to # differentiate between 350m and all other checkpoitns # Before - hidden_states = self.self_attn_layer_norm(hidden_states) + if self.do_layer_norm_before: + hidden_states = self.self_attn_layer_norm(hidden_states) hidden_states, self_attn_weights, present_key_value = self.self_attn( hidden_states=hidden_states, past_key_value=past_key_value, @@ -340,14 +342,16 @@ def forward( hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) hidden_states = residual + hidden_states # After -# hidden_states = self.self_attn_layer_norm(hidden_states) + if not self.do_layer_norm_before: + hidden_states = self.self_attn_layer_norm(hidden_states) # Fully Connected hidden_states_shape = hidden_states.shape hidden_states = hidden_states.reshape(-1, hidden_states.size(-1)) residual = hidden_states # Before - hidden_states = self.final_layer_norm(hidden_states) + if self.do_layer_norm_before: + hidden_states = self.final_layer_norm(hidden_states) hidden_states = self.fc1(hidden_states) hidden_states = self.activation_fn(hidden_states) @@ -356,7 +360,8 @@ def forward( hidden_states = (residual + hidden_states).view(hidden_states_shape) # After -# hidden_states = self.final_layer_norm(hidden_states) + if not self.do_layer_norm_before: + hidden_states = self.final_layer_norm(hidden_states) outputs = (hidden_states,) From 156866d5f9f21d020129a940f5570d24063dcf67 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Wed, 11 May 2022 13:17:30 +0200 Subject: [PATCH 088/145] solved mismatch in load dictionnary --- tests/models/opt/test_modeling_opt.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index 95ca7decc30b4..2f86f8ff1e6b9 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -78,6 +78,7 @@ def __init__( pad_token_id=1, bos_token_id=0, embed_dim=16, + word_embed_proj_dim=16 ): self.parent = parent self.batch_size = batch_size @@ -97,6 +98,7 @@ def __init__( self.pad_token_id = pad_token_id self.bos_token_id = bos_token_id self.embed_dim = embed_dim + self.word_embed_proj_dim = word_embed_proj_dim self.is_encoder_decoder = False def prepare_config_and_inputs(self): @@ -127,6 +129,7 @@ def get_config(self): pad_token_id=self.pad_token_id, embed_dim=self.embed_dim, is_encoder_decoder=False, + word_embed_proj_dim=self.word_embed_proj_dim ) def get_pipeline_config(self): From 849afd30344b26a5be7668657c990f16c4e36576 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Wed, 11 May 2022 13:24:35 +0200 Subject: [PATCH 089/145] clean up preapre opt input dict --- tests/models/opt/test_modeling_opt.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index 2f86f8ff1e6b9..2fcf483db4392 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -46,10 +46,6 @@ def prepare_opt_inputs_dict( ): if attention_mask is None: attention_mask = input_ids.ne(config.pad_token_id) - if decoder_attention_mask is None: - decoder_attention_mask = decoder_input_ids.ne(config.pad_token_id) - if decoder_head_mask is None: - decoder_head_mask = torch.ones(config.num_hidden_layers, config.num_attention_heads, device=torch_device) return { "input_ids": input_ids, "attention_mask": attention_mask, From 1acb47a0d04bddafa8d944ead0d05c1e0769913c Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Wed, 11 May 2022 13:31:59 +0200 Subject: [PATCH 090/145] fixed past key value as bool --- src/transformers/models/opt/modeling_opt.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index c7fd45f62317e..ab7393fd35275 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -327,8 +327,6 @@ def forward( residual = hidden_states # Self Attention - # TODO(ArthurZ) could you add a flag to the config "do_layer_norm_before = True/False" to - # differentiate between 350m and all other checkpoitns # Before if self.do_layer_norm_before: hidden_states = self.self_attn_layer_norm(hidden_states) @@ -700,7 +698,8 @@ def forward( def create_custom_forward(module): def custom_forward(*inputs): # None for past_key_value - return module(*inputs, output_attentions, use_cache) + print(input) + return module(*inputs, output_attentions,past_key_value) return custom_forward From 668246b1f8f4f2289ea865584d068cafb5d247f1 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Wed, 11 May 2022 13:32:08 +0200 Subject: [PATCH 091/145] fix previus --- src/transformers/models/opt/modeling_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index ab7393fd35275..83f42624af645 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -699,7 +699,7 @@ def create_custom_forward(module): def custom_forward(*inputs): # None for past_key_value print(input) - return module(*inputs, output_attentions,past_key_value) + return module(*inputs, output_attentions,None) return custom_forward From 769b9d6502277385d8bd5c575d399a7494eeb54e Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Wed, 11 May 2022 13:34:45 +0200 Subject: [PATCH 092/145] fixed return dict False tuple issue --- src/transformers/models/opt/modeling_opt.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 83f42624af645..824273d2e10fb 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -698,7 +698,6 @@ def forward( def create_custom_forward(module): def custom_forward(*inputs): # None for past_key_value - print(input) return module(*inputs, output_attentions,None) return custom_forward @@ -818,6 +817,9 @@ def forward( return_dict=return_dict, ) + if not return_dict: + return decoder_outputs + return BaseModelOutputWithPast( last_hidden_state=decoder_outputs.last_hidden_state, past_key_values=decoder_outputs.past_key_values, From def917e62608f06b8f35dfbf6874c7a7f398c2ce Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Wed, 11 May 2022 13:38:58 +0200 Subject: [PATCH 093/145] All tests are passing --- src/transformers/models/opt/modeling_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index 824273d2e10fb..cc2eb6fbd7ce7 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -723,7 +723,7 @@ def custom_forward(*inputs): hidden_states = layer_outputs[0] if use_cache: - next_decoder_cache += (layer_outputs[3 if output_attentions else 1],) + next_decoder_cache += (layer_outputs[2 if output_attentions else 1],) if output_attentions: all_self_attns += (layer_outputs[1],) From 5131932ce613ff0195870fc5d6498fa8d959d1bf Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Wed, 11 May 2022 13:42:20 +0200 Subject: [PATCH 094/145] Make style --- .../models/opt/configuration_opt.py | 2 +- ..._original_pytorch_checkpoint_to_pytorch.py | 8 ++-- src/transformers/models/opt/modeling_opt.py | 37 ++++++++++--------- tests/models/opt/test_modeling_opt.py | 7 ++-- 4 files changed, 28 insertions(+), 26 deletions(-) diff --git a/src/transformers/models/opt/configuration_opt.py b/src/transformers/models/opt/configuration_opt.py index c0af2e1c1a0df..4a9c1ea580ad3 100644 --- a/src/transformers/models/opt/configuration_opt.py +++ b/src/transformers/models/opt/configuration_opt.py @@ -109,7 +109,7 @@ def __init__( decoder_start_token_id=2, forced_eos_token_id=2, output_projection=True, - do_layer_norm_before = True, + do_layer_norm_before=True, **kwargs ): self.vocab_size = vocab_size diff --git a/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py b/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py index 60ef10af2931a..d9be3eba80970 100644 --- a/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py +++ b/src/transformers/models/opt/convert_opt_original_pytorch_checkpoint_to_pytorch.py @@ -67,11 +67,11 @@ def convert_opt_checkpoint(checkpoint_path, pytorch_dump_folder_path, config=Non parser = argparse.ArgumentParser() # Required parameters parser.add_argument( - "--fairseq_path", type=str, help="path to fairseq checkpoint in correct format. You can find all checkpoints in the correct format here: https://huggingface.co/models?other=opt_metasq" + "--fairseq_path", + type=str, + help="path to fairseq checkpoint in correct format. You can find all checkpoints in the correct format here: https://huggingface.co/models?other=opt_metasq", ) parser.add_argument("--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model.") - parser.add_argument( - "--hf_config", default=None, type=str, help="Define HF config." - ) + parser.add_argument("--hf_config", default=None, type=str, help="Define HF config.") args = parser.parse_args() convert_opt_checkpoint(args.fairseq_path, args.pytorch_dump_folder_path, config=args.hf_config) diff --git a/src/transformers/models/opt/modeling_opt.py b/src/transformers/models/opt/modeling_opt.py index cc2eb6fbd7ce7..4c4a53b2a48d1 100644 --- a/src/transformers/models/opt/modeling_opt.py +++ b/src/transformers/models/opt/modeling_opt.py @@ -328,7 +328,7 @@ def forward( # Self Attention # Before - if self.do_layer_norm_before: + if self.do_layer_norm_before: hidden_states = self.self_attn_layer_norm(hidden_states) hidden_states, self_attn_weights, present_key_value = self.self_attn( hidden_states=hidden_states, @@ -340,7 +340,7 @@ def forward( hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) hidden_states = residual + hidden_states # After - if not self.do_layer_norm_before: + if not self.do_layer_norm_before: hidden_states = self.self_attn_layer_norm(hidden_states) # Fully Connected @@ -348,7 +348,7 @@ def forward( hidden_states = hidden_states.reshape(-1, hidden_states.size(-1)) residual = hidden_states # Before - if self.do_layer_norm_before: + if self.do_layer_norm_before: hidden_states = self.final_layer_norm(hidden_states) hidden_states = self.fc1(hidden_states) hidden_states = self.activation_fn(hidden_states) @@ -358,7 +358,7 @@ def forward( hidden_states = (residual + hidden_states).view(hidden_states_shape) # After - if not self.do_layer_norm_before: + if not self.do_layer_norm_before: hidden_states = self.final_layer_norm(hidden_states) outputs = (hidden_states,) @@ -455,7 +455,8 @@ def _set_gradient_checkpointing(self, module, value=False): Indices can be obtained using [`OPTTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. - If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). + If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see + `past_key_values`). If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_inputs`] and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the @@ -535,7 +536,7 @@ def __init__(self, config: OPTConfig): self.project_in = None # Think we can delete this -# self.layer_norm = nn.LayerNorm(config.d_model) if config.decoder_layernorm else None + # self.layer_norm = nn.LayerNorm(config.d_model) if config.decoder_layernorm else None self.layer_norm = None self.layers = nn.ModuleList([OPTDecoderLayer(config) for _ in range(config.num_hidden_layers)]) @@ -650,8 +651,12 @@ def forward( inputs_embeds = self.embed_tokens(input_ids) * self.embed_scale # embed positions - attention_mask = attention_mask if attention_mask is not None else torch.ones(inputs_embeds.shape[:2], dtype=torch.bool, device=inputs_embeds.device) - positions = self.embed_positions(attention_mask)[:,past_key_values_length:,:] + attention_mask = ( + attention_mask + if attention_mask is not None + else torch.ones(inputs_embeds.shape[:2], dtype=torch.bool, device=inputs_embeds.device) + ) + positions = self.embed_positions(attention_mask)[:, past_key_values_length:, :] attention_mask = self._prepare_decoder_attention_mask( attention_mask, input_shape, inputs_embeds, past_key_values_length @@ -698,7 +703,7 @@ def forward( def create_custom_forward(module): def custom_forward(*inputs): # None for past_key_value - return module(*inputs, output_attentions,None) + return module(*inputs, output_attentions, None) return custom_forward @@ -728,8 +733,8 @@ def custom_forward(*inputs): if output_attentions: all_self_attns += (layer_outputs[1],) -# if self.layer_norm: -# hidden_states = self.layer_norm(hidden_states) + # if self.layer_norm: + # hidden_states = self.layer_norm(hidden_states) if self.project_out is not None: hidden_states = self.project_out(hidden_states) @@ -740,11 +745,7 @@ def custom_forward(*inputs): next_cache = next_decoder_cache if use_cache else None if not return_dict: - return tuple( - v - for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] - if v is not None - ) + return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None) return BaseModelOutputWithPast( last_hidden_state=hidden_states, past_key_values=next_cache, @@ -819,7 +820,7 @@ def forward( if not return_dict: return decoder_outputs - + return BaseModelOutputWithPast( last_hidden_state=decoder_outputs.last_hidden_state, past_key_values=decoder_outputs.past_key_values, @@ -934,8 +935,8 @@ def forward( ```python >>> from transformers import OPTTokenizer, OPTForCausalLM - # this needs fixing + >>> tokenizer = OPTTokenizer.from_pretrained("patrickvonplaten/opt_gpt2_tokenizer") >>> model = OPTForCausalLM.from_pretrained("ArthurZ/opt-350m") >>> assert model.config.is_decoder, f"{model.__class__} has to be configured as a decoder." diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index 2fcf483db4392..df41f9cfbfadd 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -32,6 +32,7 @@ if is_torch_available(): import torch + from transformers import GPT2Tokenizer, OPTForCausalLM, OPTModel @@ -74,7 +75,7 @@ def __init__( pad_token_id=1, bos_token_id=0, embed_dim=16, - word_embed_proj_dim=16 + word_embed_proj_dim=16, ): self.parent = parent self.batch_size = batch_size @@ -125,7 +126,7 @@ def get_config(self): pad_token_id=self.pad_token_id, embed_dim=self.embed_dim, is_encoder_decoder=False, - word_embed_proj_dim=self.word_embed_proj_dim + word_embed_proj_dim=self.word_embed_proj_dim, ) def get_pipeline_config(self): @@ -139,7 +140,7 @@ def prepare_config_and_inputs_for_common(self): def create_and_check_decoder_model_past_large_inputs(self, config, inputs_dict): model = OPTModel(config=config).to(torch_device).eval() - + input_ids = inputs_dict["input_ids"] attention_mask = inputs_dict["attention_mask"] head_mask = inputs_dict["head_mask"] From 5ec07666f97e323282ed38e9773dc361ff758f6f Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Wed, 11 May 2022 13:49:17 +0200 Subject: [PATCH 095/145] Ignore OPTDecoder non tested --- utils/check_repo.py | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/check_repo.py b/utils/check_repo.py index a1bdae71929c4..b75275f86fd6c 100644 --- a/utils/check_repo.py +++ b/utils/check_repo.py @@ -45,6 +45,7 @@ # Being in this list is an exception and should **not** be the rule. IGNORE_NON_TESTED = PRIVATE_MODELS.copy() + [ # models to ignore for not tested + "OPTDecoder" "DecisionTransformerGPT2Model", # Building part of bigger (tested) model. "SegformerDecodeHead", # Building part of bigger (tested) model. "PLBartEncoder", # Building part of bigger (tested) model. From 2ed32a85677c2e1b880ec220454ced1efb592b98 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Wed, 11 May 2022 13:51:10 +0200 Subject: [PATCH 096/145] make fix-copies --- docs/source/en/index.mdx | 2 +- docs/source/en/serialization.mdx | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/source/en/index.mdx b/docs/source/en/index.mdx index f373ce399ec8a..a1808e4b1ae01 100644 --- a/docs/source/en/index.mdx +++ b/docs/source/en/index.mdx @@ -229,7 +229,7 @@ Flax), PyTorch, and/or TensorFlow. | Nystromformer | ❌ | ❌ | ✅ | ❌ | ❌ | | OpenAI GPT | ✅ | ✅ | ✅ | ✅ | ❌ | | OpenAI GPT-2 | ✅ | ✅ | ✅ | ✅ | ✅ | -| OPT | ❌ | ❌ | ✅ | ✅ | ✅ | +| OPT | ❌ | ❌ | ✅ | ❌ | ❌ | | Pegasus | ✅ | ✅ | ✅ | ✅ | ✅ | | Perceiver | ✅ | ❌ | ✅ | ❌ | ❌ | | PLBart | ✅ | ❌ | ✅ | ❌ | ❌ | diff --git a/docs/source/en/serialization.mdx b/docs/source/en/serialization.mdx index e069e6c5411b9..df39f1b97c547 100644 --- a/docs/source/en/serialization.mdx +++ b/docs/source/en/serialization.mdx @@ -70,7 +70,6 @@ Ready-made configurations include the following architectures: - mBART - MobileBERT - OpenAI GPT-2 -- OPT - PLBart - RoBERTa - RoFormer From 1db5f2bfe357afdd9d0a4e40167c9ae34f08f242 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Wed, 11 May 2022 13:53:23 +0200 Subject: [PATCH 097/145] make repo consistency --- utils/check_repo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/check_repo.py b/utils/check_repo.py index b75275f86fd6c..8fec4635aa427 100644 --- a/utils/check_repo.py +++ b/utils/check_repo.py @@ -45,7 +45,7 @@ # Being in this list is an exception and should **not** be the rule. IGNORE_NON_TESTED = PRIVATE_MODELS.copy() + [ # models to ignore for not tested - "OPTDecoder" + "OPTDecoder", # Building part of bigger (tested) model. "DecisionTransformerGPT2Model", # Building part of bigger (tested) model. "SegformerDecodeHead", # Building part of bigger (tested) model. "PLBartEncoder", # Building part of bigger (tested) model. From f57a0b5939475fdf010a2a5e7fe30603101a29dc Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Wed, 11 May 2022 14:00:09 +0200 Subject: [PATCH 098/145] small fix --- utils/check_repo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/check_repo.py b/utils/check_repo.py index 8fec4635aa427..534e1d8b4046a 100644 --- a/utils/check_repo.py +++ b/utils/check_repo.py @@ -45,7 +45,7 @@ # Being in this list is an exception and should **not** be the rule. IGNORE_NON_TESTED = PRIVATE_MODELS.copy() + [ # models to ignore for not tested - "OPTDecoder", # Building part of bigger (tested) model. + "OPTDecoder", # Building part of bigger (tested) model. "DecisionTransformerGPT2Model", # Building part of bigger (tested) model. "SegformerDecodeHead", # Building part of bigger (tested) model. "PLBartEncoder", # Building part of bigger (tested) model. From 5f96836d0c01617b53162617d5246e7d22fea757 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Wed, 11 May 2022 14:12:25 +0200 Subject: [PATCH 099/145] removed uselss @torch.no_grad decorator --- tests/models/opt/test_modeling_opt.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index df41f9cfbfadd..db9ab9d15074c 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -303,8 +303,6 @@ def test_load_model(self): except BaseException: self.fail("Failed loading model") - @require_torch - @torch.no_grad() def test_logits(self): model = OPTForCausalLM.from_pretrained(self.path_model) model = model.eval() @@ -338,9 +336,12 @@ def test_logits(self): class OPTGenerationTest(unittest.TestCase): def setUp(self): super().setUp() - self.path_model = "ArthurZ/opt-350m" - # self.path_logits_meta = "/home/younes/Desktop/Work/metaseq-conversion/logits_metaseq_gpt2_tokenizer.p" # TODO add the logits somewhere? - + self.all_model_path = [ + "facebook/opt-125m", + "facebook/opt-350m", + "facebook/opt-1.3b" + ] + def test_generation(self): model = OPTForCausalLM.from_pretrained(self.path_model) model = model.eval() From 70c219623c8a4ecbe769e07d41a96a4964176391 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Wed, 11 May 2022 14:13:00 +0200 Subject: [PATCH 100/145] make styl;e --- tests/models/opt/test_modeling_opt.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index db9ab9d15074c..d80a553ef3a5e 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -336,12 +336,8 @@ def test_logits(self): class OPTGenerationTest(unittest.TestCase): def setUp(self): super().setUp() - self.all_model_path = [ - "facebook/opt-125m", - "facebook/opt-350m", - "facebook/opt-1.3b" - ] - + self.all_model_path = ["facebook/opt-125m", "facebook/opt-350m", "facebook/opt-1.3b"] + def test_generation(self): model = OPTForCausalLM.from_pretrained(self.path_model) model = model.eval() From 49e905d17995877d9c979c078ceb40eca1d6d977 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Wed, 11 May 2022 14:15:58 +0200 Subject: [PATCH 101/145] fix previous opt test --- tests/models/opt/test_modeling_opt.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index d80a553ef3a5e..96ffd38516c76 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -336,8 +336,8 @@ def test_logits(self): class OPTGenerationTest(unittest.TestCase): def setUp(self): super().setUp() - self.all_model_path = ["facebook/opt-125m", "facebook/opt-350m", "facebook/opt-1.3b"] - + # self.all_model_path = ["facebook/opt-125m", "facebook/opt-350m", "facebook/opt-1.3b"] + self.path_model = "ArthurZ/opt-350m" def test_generation(self): model = OPTForCausalLM.from_pretrained(self.path_model) model = model.eval() From 2c1bce4e1bde8648e4d1cf1f536ec2334660de04 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Wed, 11 May 2022 14:16:22 +0200 Subject: [PATCH 102/145] style --- tests/models/opt/test_modeling_opt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index 96ffd38516c76..316bbc2167c8a 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -338,6 +338,7 @@ def setUp(self): super().setUp() # self.all_model_path = ["facebook/opt-125m", "facebook/opt-350m", "facebook/opt-1.3b"] self.path_model = "ArthurZ/opt-350m" + def test_generation(self): model = OPTForCausalLM.from_pretrained(self.path_model) model = model.eval() From 9c3f0c0ddf45da17c8231a1ab5413e097c962415 Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Wed, 11 May 2022 14:16:28 +0200 Subject: [PATCH 103/145] make style --- tests/models/opt/test_modeling_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/models/opt/test_modeling_opt.py b/tests/models/opt/test_modeling_opt.py index 316bbc2167c8a..e952e787f6011 100644 --- a/tests/models/opt/test_modeling_opt.py +++ b/tests/models/opt/test_modeling_opt.py @@ -338,7 +338,7 @@ def setUp(self): super().setUp() # self.all_model_path = ["facebook/opt-125m", "facebook/opt-350m", "facebook/opt-1.3b"] self.path_model = "ArthurZ/opt-350m" - + def test_generation(self): model = OPTForCausalLM.from_pretrained(self.path_model) model = model.eval() From 29987ed5dadf68fca9d745d76ffb5fd61c71996c Mon Sep 17 00:00:00 2001 From: ArthurZucker Date: Wed, 11 May 2022 14:49:20 +0200 Subject: [PATCH 104/145] added opt documentation --- docs/source/en/model_doc/opt.mdx | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/source/en/model_doc/opt.mdx b/docs/source/en/model_doc/opt.mdx index 6cda45da86045..a2c3d35dc9b63 100644 --- a/docs/source/en/model_doc/opt.mdx +++ b/docs/source/en/model_doc/opt.mdx @@ -1,4 +1,4 @@ -