From edda5373a2eb12d04d15fb74e9a18526a8a4d839 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Thu, 8 Dec 2022 14:28:18 +0000 Subject: [PATCH 01/96] add new model like --- docs/source/en/model_doc/blip.mdx | 71 + src/transformers/__init__.py | 36 + src/transformers/models/__init__.py | 1 + .../models/auto/configuration_auto.py | 3 + .../models/auto/feature_extraction_auto.py | 1 + .../models/auto/image_processing_auto.py | 1 + src/transformers/models/auto/modeling_auto.py | 2 + .../models/auto/processing_auto.py | 1 + .../models/auto/tokenization_auto.py | 6 + src/transformers/models/blip/__init__.py | 81 + .../models/blip/configuration_blip.py | 404 +++++ .../convert_blip_original_pytorch_to_hf.py | 148 ++ src/transformers/models/blip/modeling_blip.py | 1331 +++++++++++++++++ tests/models/blip/__init__.py | 0 tests/models/blip/test_modeling_blip.py | 737 +++++++++ 15 files changed, 2823 insertions(+) create mode 100644 docs/source/en/model_doc/blip.mdx create mode 100644 src/transformers/models/blip/__init__.py create mode 100644 src/transformers/models/blip/configuration_blip.py create mode 100644 src/transformers/models/blip/convert_blip_original_pytorch_to_hf.py create mode 100644 src/transformers/models/blip/modeling_blip.py create mode 100644 tests/models/blip/__init__.py create mode 100644 tests/models/blip/test_modeling_blip.py diff --git a/docs/source/en/model_doc/blip.mdx b/docs/source/en/model_doc/blip.mdx new file mode 100644 index 000000000000..468f80b07e9f --- /dev/null +++ b/docs/source/en/model_doc/blip.mdx @@ -0,0 +1,71 @@ + + +# BLIP + +## Overview + +The BLIP model was proposed in []() by . + + +The abstract from the paper is the following: + +** + +Tips: + + + +This model was contributed by [INSERT YOUR HF USERNAME HERE](https://huggingface.co/). +The original code can be found [here](). + + +## BLIPConfig + +[[autodoc]] BLIPConfig + - from_text_vision_configs + +## BLIPTextConfig + +[[autodoc]] BLIPTextConfig + +## BLIPVisionConfig + +[[autodoc]] BLIPVisionConfig + +## BLIPModel + +[[autodoc]] BLIPModel + - forward + - get_text_features + - get_image_features + +## BLIPTextModel + +[[autodoc]] BLIPTextModel + - forward + +## BLIPTextModelWithProjection + +[[autodoc]] BLIPTextModelWithProjection + - forward + +## BLIPVisionModelWithProjection + +[[autodoc]] BLIPVisionModelWithProjection + - forward + + +## BLIPVisionModel + +[[autodoc]] BLIPVisionModel + - forward diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index dce1d73e9fad..faf996c997c3 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -188,6 +188,14 @@ "CLIPTokenizer", "CLIPVisionConfig", ], + "models.blip": [ + "BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP", + "BLIPConfig", + + "BLIPTextConfig", + + "BLIPVisionConfig", + ], "models.clipseg": [ "CLIPSEG_PRETRAINED_CONFIG_ARCHIVE_MAP", "CLIPSegConfig", @@ -1143,6 +1151,17 @@ "CLIPVisionModelWithProjection", ] ) + _import_structure["models.blip"].extend( + [ + "BLIP_PRETRAINED_MODEL_ARCHIVE_LIST", + "BLIPModel", + "BLIPPreTrainedModel", + "BLIPTextModel", + "BLIPTextModelWithProjection", + "BLIPVisionModel", + "BLIPVisionModelWithProjection", + ] + ) _import_structure["models.clipseg"].extend( [ "CLIPSEG_PRETRAINED_MODEL_ARCHIVE_LIST", @@ -3451,6 +3470,14 @@ CLIPTokenizer, CLIPVisionConfig, ) + from .models.blip import ( + BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP, + BLIPConfig, + + BLIPTextConfig, + + BLIPVisionConfig, + ) from .models.clipseg import ( CLIPSEG_PRETRAINED_CONFIG_ARCHIVE_MAP, CLIPSegConfig, @@ -4277,6 +4304,15 @@ CLIPVisionModel, CLIPVisionModelWithProjection, ) + from .models.blip import ( + BLIP_PRETRAINED_MODEL_ARCHIVE_LIST, + BLIPModel, + BLIPPreTrainedModel, + BLIPTextModel, + BLIPTextModelWithProjection, + BLIPVisionModel, + BLIPVisionModelWithProjection, + ) from .models.clipseg import ( CLIPSEG_PRETRAINED_MODEL_ARCHIVE_LIST, CLIPSegForImageSegmentation, diff --git a/src/transformers/models/__init__.py b/src/transformers/models/__init__.py index d81ca5ac828c..fd645f3e294f 100644 --- a/src/transformers/models/__init__.py +++ b/src/transformers/models/__init__.py @@ -41,6 +41,7 @@ canine, chinese_clip, clip, + blip, clipseg, codegen, conditional_detr, diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index bcfc7bdde481..8737454063c7 100644 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -46,6 +46,7 @@ ("canine", "CanineConfig"), ("chinese_clip", "ChineseCLIPConfig"), ("clip", "CLIPConfig"), + ("blip", "BLIPConfig"), ("clipseg", "CLIPSegConfig"), ("codegen", "CodeGenConfig"), ("conditional_detr", "ConditionalDetrConfig"), @@ -201,6 +202,7 @@ ("canine", "CANINE_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("chinese_clip", "CHINESE_CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("clip", "CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP"), + ("blip", "BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("clipseg", "CLIPSEG_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("codegen", "CODEGEN_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("conditional_detr", "CONDITIONAL_DETR_PRETRAINED_CONFIG_ARCHIVE_MAP"), @@ -348,6 +350,7 @@ ("canine", "CANINE"), ("chinese_clip", "Chinese-CLIP"), ("clip", "CLIP"), + ("blip", "BLIP"), ("clipseg", "CLIPSeg"), ("codegen", "CodeGen"), ("conditional_detr", "Conditional DETR"), diff --git a/src/transformers/models/auto/feature_extraction_auto.py b/src/transformers/models/auto/feature_extraction_auto.py index a33affe3ec9b..8a30011b543c 100644 --- a/src/transformers/models/auto/feature_extraction_auto.py +++ b/src/transformers/models/auto/feature_extraction_auto.py @@ -41,6 +41,7 @@ ("beit", "BeitFeatureExtractor"), ("chinese_clip", "ChineseCLIPFeatureExtractor"), ("clip", "CLIPFeatureExtractor"), + ("blip", "BLIPFeatureExtractor"), ("clipseg", "ViTFeatureExtractor"), ("conditional_detr", "ConditionalDetrFeatureExtractor"), ("convnext", "ConvNextFeatureExtractor"), diff --git a/src/transformers/models/auto/image_processing_auto.py b/src/transformers/models/auto/image_processing_auto.py index ea08c0fe8dc5..31aa17ec1be4 100644 --- a/src/transformers/models/auto/image_processing_auto.py +++ b/src/transformers/models/auto/image_processing_auto.py @@ -41,6 +41,7 @@ ("bit", "BitImageProcessor"), ("chinese_clip", "ChineseCLIPImageProcessor"), ("clip", "CLIPImageProcessor"), + ("blip", "BLIPImageProcessor"), ("clipseg", "ViTImageProcessor"), ("conditional_detr", "ConditionalDetrImageProcessor"), ("convnext", "ConvNextImageProcessor"), diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index 63453438c893..a1b36020a2eb 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -45,6 +45,7 @@ ("canine", "CanineModel"), ("chinese_clip", "ChineseCLIPModel"), ("clip", "CLIPModel"), + ("blip", "BLIPModel"), ("clipseg", "CLIPSegModel"), ("codegen", "CodeGenModel"), ("conditional_detr", "ConditionalDetrModel"), @@ -851,6 +852,7 @@ # Model for Zero Shot Image Classification mapping ("chinese_clip", "ChineseCLIPModel"), ("clip", "CLIPModel"), + ("blip", "BLIPModel"), ("clipseg", "CLIPSegModel"), ] ) diff --git a/src/transformers/models/auto/processing_auto.py b/src/transformers/models/auto/processing_auto.py index 9399c09be655..f72b481a2ebe 100644 --- a/src/transformers/models/auto/processing_auto.py +++ b/src/transformers/models/auto/processing_auto.py @@ -43,6 +43,7 @@ [ ("chinese_clip", "ChineseCLIPProcessor"), ("clip", "CLIPProcessor"), + ("blip", "BLIPProcessor"), ("clipseg", "CLIPSegProcessor"), ("flava", "FlavaProcessor"), ("groupvit", "CLIPProcessor"), diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index e2fa1ddd7ce6..43c3c2af3444 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -95,6 +95,12 @@ "CLIPTokenizerFast" if is_tokenizers_available() else None, ), ), + ( + "blip", + ( + "CLIPTokenizer", + "CLIPTokenizerFast" if is_tokenizers_available() else None, + ), ( "clipseg", ( diff --git a/src/transformers/models/blip/__init__.py b/src/transformers/models/blip/__init__.py new file mode 100644 index 000000000000..92f07d52b53e --- /dev/null +++ b/src/transformers/models/blip/__init__.py @@ -0,0 +1,81 @@ +# flake8: noqa +# There's no way to ignore "F401 '...' imported but unused" warnings in this +# module, but to preserve other warnings. So, don't check this module at all. + +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +from ...utils import ( + OptionalDependencyNotAvailable, + _LazyModule, + is_torch_available, +) + + +_import_structure = { + "configuration_blip": [ + "BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP", + "BLIPConfig", + "BLIPOnnxConfig", + "BLIPTextConfig", + "BLIPVisionConfig", + ], +} + +try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + pass +else: + _import_structure["modeling_blip"] = [ + "BLIP_PRETRAINED_MODEL_ARCHIVE_LIST", + "BLIPModel", + "BLIPPreTrainedModel", + "BLIPTextModel", + "BLIPTextModelWithProjection", + "BLIPVisionModel", + "BLIPVisionModelWithProjection", + ] + +if TYPE_CHECKING: + from .configuration_blip import ( + BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP, + BLIPConfig, + BLIPOnnxConfig, + BLIPTextConfig, + BLIPVisionConfig, + ) + + try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + pass + else: + from .modeling_blip import ( + BLIP_PRETRAINED_MODEL_ARCHIVE_LIST, + BLIPModel, + BLIPPreTrainedModel, + BLIPTextModel, + BLIPTextModelWithProjection, + BLIPVisionModel, + BLIPVisionModelWithProjection, + ) + +else: + import sys + + sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__) diff --git a/src/transformers/models/blip/configuration_blip.py b/src/transformers/models/blip/configuration_blip.py new file mode 100644 index 000000000000..08da2af6eee2 --- /dev/null +++ b/src/transformers/models/blip/configuration_blip.py @@ -0,0 +1,404 @@ +# coding=utf-8 +# Copyright 2022 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" BLIP model configuration""" + +import copy +import os +from collections import OrderedDict +from typing import TYPE_CHECKING, Any, Mapping, Optional, Union + + +if TYPE_CHECKING: + from ...processing_utils import ProcessorMixin + from ...utils import TensorType + +from ...configuration_utils import PretrainedConfig +from ...onnx import OnnxConfig +from ...utils import logging + + +logger = logging.get_logger(__name__) + +BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP = { + "ybelkada/blip-base": "https://huggingface.co/ybelkada/blip-base/resolve/main/config.json", +} + + + +class BLIPTextConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`BLIPModel`]. It is used to instantiate an BLIP + model according to the specified arguments, defining the model architecture. Instantiating a configuration with the + defaults will yield a similar configuration to that of the BLIP + [ybelkada/blip-base](https://huggingface.co/ybelkada/blip-base) architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + vocab_size (`int`, *optional*, defaults to 49408): + Vocabulary size of the BLIP text model. Defines the number of different tokens that can be represented by + the `inputs_ids` passed when calling [`BLIPModel`]. + hidden_size (`int`, *optional*, defaults to 512): + Dimensionality of the encoder layers and the pooler layer. + intermediate_size (`int`, *optional*, defaults to 2048): + Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. + num_hidden_layers (`int`, *optional*, defaults to 12): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 8): + Number of attention heads for each attention layer in the Transformer encoder. + max_position_embeddings (`int`, *optional*, defaults to 77): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported. layer_norm_eps (`float`, *optional*, + defaults to 1e-5): The epsilon used by the layer normalization layers. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + dropout (`float`, *optional*, defaults to 0.0): + The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + initializer_factor (`float``, *optional*, defaults to 1): + A factor for initializing all weight matrices (should be kept to 1, used internally for initialization + testing). + + Example: + + ```python + >>> from transformers import BLIPTextConfig, BLIPTextModel + + >>> # Initializing a BLIPTextConfig with ybelkada/blip-base style configuration + >>> configuration = BLIPTextConfig() + + >>> # Initializing a BLIPTextModel (with random weights) from the ybelkada/blip-base style configuration + >>> model = BLIPTextModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + model_type = "blip_text_model" + + def __init__( + self, + vocab_size=49408, + hidden_size=512, + intermediate_size=2048, + projection_dim=512, + num_hidden_layers=12, + num_attention_heads=8, + max_position_embeddings=77, + hidden_act="quick_gelu", + layer_norm_eps=0.00001, + dropout=0.0, + attention_dropout=0.0, + initializer_range=0.02, + initializer_factor=1.0, + pad_token_id=1, + bos_token_id=0, + eos_token_id=2, + **kwargs + ): + super().__init__(pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs) + + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.projection_dim = projection_dim + self.dropout = dropout + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.max_position_embeddings = max_position_embeddings + self.layer_norm_eps = layer_norm_eps + self.hidden_act = hidden_act + self.initializer_range = initializer_range + self.initializer_factor = initializer_factor + self.attention_dropout = attention_dropout + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) + + # get the text config dict if we are loading from BLIPConfig + if config_dict.get("model_type") == "blip": + config_dict = config_dict["text_config"] + + if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type: + logger.warning( + f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " + f"{cls.model_type}. This is not supported for all configurations of models and can yield errors." + ) + + return cls.from_dict(config_dict, **kwargs) + + +class BLIPVisionConfig(PretrainedConfig): + r""" + This is the configuration class to store the configuration of a [`BLIPModel`]. It is used to instantiate an BLIP + model according to the specified arguments, defining the model architecture. Instantiating a configuration with the + defaults will yield a similar configuration to that of the BLIP + [ybelkada/blip-base](https://huggingface.co/ybelkada/blip-base) architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + + Args: + hidden_size (`int`, *optional*, defaults to 768): + Dimensionality of the encoder layers and the pooler layer. + intermediate_size (`int`, *optional*, defaults to 3072): + Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. + num_hidden_layers (`int`, *optional*, defaults to 12): + Number of hidden layers in the Transformer encoder. + num_attention_heads (`int`, *optional*, defaults to 12): + Number of attention heads for each attention layer in the Transformer encoder. + image_size (`int`, *optional*, defaults to 224): + The size (resolution) of each image. + patch_size (`int`, *optional*, defaults to 32): + The size (resolution) of each patch. + hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`): + The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, + `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported. layer_norm_eps (`float`, *optional*, + defaults to 1e-5): The epsilon used by the layer normalization layers. + dropout (`float`, *optional*, defaults to 0.0): + The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler. + attention_dropout (`float`, *optional*, defaults to 0.0): + The dropout ratio for the attention probabilities. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + initializer_factor (`float``, *optional*, defaults to 1): + A factor for initializing all weight matrices (should be kept to 1, used internally for initialization + testing). + + Example: + + ```python + >>> from transformers import BLIPVisionConfig, BLIPVisionModel + + >>> # Initializing a BLIPVisionConfig with ybelkada/blip-base style configuration + >>> configuration = BLIPVisionConfig() + + >>> # Initializing a BLIPVisionModel (with random weights) from the ybelkada/blip-base style configuration + >>> model = BLIPVisionModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + ```""" + + model_type = "blip_vision_model" + + def __init__( + self, + hidden_size=768, + intermediate_size=3072, + projection_dim=512, + num_hidden_layers=12, + num_attention_heads=12, + num_channels=3, + image_size=224, + patch_size=32, + hidden_act="quick_gelu", + layer_norm_eps=0.00001, + dropout=0.0, + attention_dropout=0.0, + initializer_range=0.02, + initializer_factor=1.0, + **kwargs + ): + super().__init__(**kwargs) + + self.hidden_size = hidden_size + self.intermediate_size = intermediate_size + self.projection_dim = projection_dim + self.dropout = dropout + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.num_channels = num_channels + self.patch_size = patch_size + self.image_size = image_size + self.initializer_range = initializer_range + self.initializer_factor = initializer_factor + self.attention_dropout = attention_dropout + self.layer_norm_eps = layer_norm_eps + self.hidden_act = hidden_act + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": + + config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) + + # get the vision config dict if we are loading from BLIPConfig + if config_dict.get("model_type") == "blip": + config_dict = config_dict["vision_config"] + + if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type: + logger.warning( + f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " + f"{cls.model_type}. This is not supported for all configurations of models and can yield errors." + ) + + return cls.from_dict(config_dict, **kwargs) + + +class BLIPConfig(PretrainedConfig): + r""" + [`BLIPConfig`] is the configuration class to store the configuration of a [`BLIPModel`]. It is used to instantiate + BLIP model according to the specified arguments, defining the text model and vision model configs. Instantiating a + configuration with the defaults will yield a similar configuration to that of the BLIP + [ybelkada/blip-base](https://huggingface.co/ybelkada/blip-base) architecture. + + Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the + documentation from [`PretrainedConfig`] for more information. + + Args: + text_config (`dict`, *optional*): + Dictionary of configuration options used to initialize [`BLIPTextConfig`]. + vision_config (`dict`, *optional*): + Dictionary of configuration options used to initialize [`BLIPVisionConfig`]. + projection_dim (`int`, *optional*, defaults to 512): + Dimentionality of text and vision projection layers. + logit_scale_init_value (`float`, *optional*, defaults to 2.6592): + The inital value of the *logit_scale* paramter. Default is used as per the original BLIP implementation. + kwargs (*optional*): + Dictionary of keyword arguments. + + Example: + + ```python + >>> from transformers import BLIPConfig, BLIPModel + + >>> # Initializing a BLIPConfig with ybelkada/blip-base style configuration + >>> configuration = BLIPConfig() + + >>> # Initializing a BLIPModel (with random weights) from the ybelkada/blip-base style configuration + >>> model = BLIPModel(configuration) + + >>> # Accessing the model configuration + >>> configuration = model.config + + >>> # We can also initialize a BLIPConfig from a BLIPTextConfig and a BLIPVisionConfig + + >>> # Initializing a BLIPText and BLIPVision configuration + >>> config_text = BLIPTextConfig() + >>> config_vision = BLIPVisionConfig() + + >>> config = BLIPConfig.from_text_vision_configs(config_text, config_vision) + ```""" + + model_type = "blip" + is_composition = True + + def __init__( + self, text_config=None, vision_config=None, projection_dim=512, logit_scale_init_value=2.6592, **kwargs + ): + super().__init__(**kwargs) + + # If `_config_dict` exist, we use them for the backward compatibility. + text_config_dict = kwargs.pop("text_config_dict", None) + vision_config_dict = kwargs.pop("vision_config_dict", None) + if text_config_dict is not None: + text_config = text_config_dict + if vision_config_dict is not None: + vision_config = vision_config_dict + + if text_config is None: + text_config = {} + logger.info("text_config is None. Initializing the BLIPTextConfig with default values.") + + if vision_config is None: + vision_config = {} + logger.info("vision_config is None. initializing the BLIPVisionConfig with default values.") + + self.text_config = BLIPTextConfig(**text_config) + self.vision_config = BLIPVisionConfig(**vision_config) + + self.projection_dim = projection_dim + self.logit_scale_init_value = logit_scale_init_value + self.initializer_factor = 1.0 + + @classmethod + def from_text_vision_configs(cls, text_config: BLIPTextConfig, vision_config: BLIPVisionConfig, **kwargs): + r""" + Instantiate a [`BLIPConfig`] (or a derived class) from blip text model configuration and blip vision model + configuration. + + Returns: + [`BLIPConfig`]: An instance of a configuration object + """ + + return cls(text_config=text_config.to_dict(), vision_config=vision_config.to_dict(), **kwargs) + + def to_dict(self): + """ + Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`]. + + Returns: + `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance, + """ + output = copy.deepcopy(self.__dict__) + output["text_config"] = self.text_config.to_dict() + output["vision_config"] = self.vision_config.to_dict() + output["model_type"] = self.__class__.model_type + return output + + +class BLIPOnnxConfig(OnnxConfig): + @property + def inputs(self) -> Mapping[str, Mapping[int, str]]: + return OrderedDict( + [ + ("input_ids", {0: "batch", 1: "sequence"}), + ("pixel_values", {0: "batch", 1: "num_channels", 2: "height", 3: "width"}), + ("attention_mask", {0: "batch", 1: "sequence"}), + ] + ) + + @property + def outputs(self) -> Mapping[str, Mapping[int, str]]: + return OrderedDict( + [ + ("logits_per_image", {0: "batch"}), + ("logits_per_text", {0: "batch"}), + ("text_embeds", {0: "batch"}), + ("image_embeds", {0: "batch"}), + ] + ) + + @property + def atol_for_validation(self) -> float: + return 1e-4 + + def generate_dummy_inputs( + self, + processor: "ProcessorMixin", + batch_size: int = -1, + seq_length: int = -1, + framework: Optional["TensorType"] = None, + ) -> Mapping[str, Any]: + + text_input_dict = super().generate_dummy_inputs( + processor.tokenizer, batch_size=batch_size, seq_length=seq_length, framework=framework + ) + image_input_dict = super().generate_dummy_inputs( + processor.feature_extractor, batch_size=batch_size, framework=framework + ) + return {**text_input_dict, **image_input_dict} + + @property + def default_onnx_opset(self) -> int: + return 14 diff --git a/src/transformers/models/blip/convert_blip_original_pytorch_to_hf.py b/src/transformers/models/blip/convert_blip_original_pytorch_to_hf.py new file mode 100644 index 000000000000..e99ed2c3af9f --- /dev/null +++ b/src/transformers/models/blip/convert_blip_original_pytorch_to_hf.py @@ -0,0 +1,148 @@ +# coding=utf-8 +# Copyright 2022 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse + +import torch + +from blip import load +from transformers import BLIPConfig, BLIPModel + + +def copy_attn_layer(hf_attn_layer, pt_attn_layer): + q_proj, k_proj, v_proj = pt_attn_layer.in_proj_weight.chunk(3, dim=0) + q_proj_bias, k_proj_bias, v_proj_bias = pt_attn_layer.in_proj_bias.chunk(3, dim=0) + + out_proj_weights = pt_attn_layer.out_proj.weight + out_proj_bias = pt_attn_layer.out_proj.bias + + hf_attn_layer.q_proj.weight.data = q_proj + hf_attn_layer.q_proj.bias.data = q_proj_bias + + hf_attn_layer.k_proj.weight.data = k_proj + hf_attn_layer.k_proj.bias.data = k_proj_bias + + hf_attn_layer.v_proj.weight.data = v_proj + hf_attn_layer.v_proj.bias.data = v_proj_bias + + hf_attn_layer.out_proj.weight = out_proj_weights + hf_attn_layer.out_proj.bias = out_proj_bias + + +def copy_mlp(hf_mlp, pt_mlp): + copy_linear(hf_mlp.fc1, pt_mlp.c_fc) + copy_linear(hf_mlp.fc2, pt_mlp.c_proj) + + +def copy_linear(hf_linear, pt_linear): + hf_linear.weight = pt_linear.weight + hf_linear.bias = pt_linear.bias + + +def copy_layer(hf_layer, pt_layer): + # copy layer norms + copy_linear(hf_layer.layer_norm1, pt_layer.ln_1) + copy_linear(hf_layer.layer_norm2, pt_layer.ln_2) + + # copy MLP + copy_mlp(hf_layer.mlp, pt_layer.mlp) + + # copy attn + copy_attn_layer(hf_layer.self_attn, pt_layer.attn) + + +def copy_layers(hf_layers, pt_layers): + for hf_layer, pt_layer in zip(hf_layers, pt_layers): + copy_layer(hf_layer, pt_layer) + + +def copy_encoder(hf_encoder, pt_model): + # copy embeds + hf_encoder.embeddings.token_embedding.weight = pt_model.token_embedding.weight + hf_encoder.embeddings.position_embedding.weight.data = pt_model.positional_embedding + + # copy layer norm + copy_linear(hf_encoder.final_layer_norm, pt_model.ln_final) + + # copy hidden layers + copy_layers(hf_encoder.encoder.layers, pt_model.transformer.resblocks) + + +def copy_text_model_and_projection(hf_model, pt_model): + # copy projection + hf_model.text_projection.weight.data = pt_model.text_projection.data.T + + # copy text encoder + copy_encoder(hf_model.text_model, pt_model) + + +def copy_vison_model_and_projection(hf_model, pt_model): + # copy projection + hf_model.visual_projection.weight.data = pt_model.visual.proj.data.T + + # copy layer norms + copy_linear(hf_model.vision_model.pre_layrnorm, pt_model.visual.ln_pre) + copy_linear(hf_model.vision_model.post_layernorm, pt_model.visual.ln_post) + + # copy embeds + hf_model.vision_model.embeddings.patch_embedding.weight.data = pt_model.visual.conv1.weight.data + hf_model.vision_model.embeddings.class_embedding = pt_model.visual.class_embedding + hf_model.vision_model.embeddings.position_embedding.weight.data = pt_model.visual.positional_embedding.data + + # copy encoder + copy_layers(hf_model.vision_model.encoder.layers, pt_model.visual.transformer.resblocks) + + +@torch.no_grad() +def convert_blip_checkpoint(checkpoint_path, pytorch_dump_folder_path, config_path=None): + """ + Copy/paste/tweak model's weights to transformers design. + """ + if config_path is not None: + config = BLIPConfig.from_pretrained(config_path) + else: + config = BLIPConfig(projection_dim=512, text_config={}, vision_config={}) + + hf_model = BLIPModel(config).eval() + + pt_model, _ = load(checkpoint_path, device="cpu", jit=False) + pt_model = pt_model.eval() + + copy_text_model_and_projection(hf_model, pt_model) + copy_vison_model_and_projection(hf_model, pt_model) + hf_model.logit_scale = pt_model.logit_scale + + input_ids = torch.arange(0, 77).unsqueeze(0) + pixel_values = torch.randn(1, 3, 224, 224) + + hf_logits_per_image, hf_logits_per_text = hf_model( + input_ids=input_ids, pixel_values=pixel_values, return_dict=True + )[1:3] + pt_logits_per_image, pt_logits_per_text = pt_model(pixel_values, input_ids) + + assert torch.allclose(hf_logits_per_image, pt_logits_per_image, atol=1e-3) + assert torch.allclose(hf_logits_per_text, pt_logits_per_text, atol=1e-3) + + hf_model.save_pretrained(pytorch_dump_folder_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model.") + parser.add_argument("--checkpoint_path", default=None, type=str, help="Path to fairseq checkpoint") + parser.add_argument("--config_path", default=None, type=str, help="Path to hf config.json of model to convert") + args = parser.parse_args() + + convert_blip_checkpoint(args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py new file mode 100644 index 000000000000..4a27654190c9 --- /dev/null +++ b/src/transformers/models/blip/modeling_blip.py @@ -0,0 +1,1331 @@ +# coding=utf-8 +# Copyright 2022 The OpenAI Team Authors and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch BLIP model.""" + + +from dataclasses import dataclass +from typing import Any, Optional, Tuple, Union + +import torch +import torch.utils.checkpoint +from torch import nn + +from ...activations import ACT2FN +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling +from ...modeling_utils import PreTrainedModel +from ...utils import ( + ModelOutput, + add_start_docstrings, + add_start_docstrings_to_model_forward, + logging, + replace_return_docstrings, +) +from .configuration_blip import BLIPConfig, BLIPTextConfig, BLIPVisionConfig + + +logger = logging.get_logger(__name__) + +_CHECKPOINT_FOR_DOC = "ybelkada/blip-base" + +BLIP_PRETRAINED_MODEL_ARCHIVE_LIST = [ + "ybelkada/blip-base", + # See all BLIP models at https://huggingface.co/models?filter=blip +] + + + +# Copied from transformers.models.bart.modeling_bart._expand_mask +def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): + """ + Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. + """ + bsz, src_len = mask.size() + tgt_len = tgt_len if tgt_len is not None else src_len + + expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype) + + inverted_mask = 1.0 - expanded_mask + + return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min) + + +# contrastive loss function, adapted from +# https://sachinruk.github.io/blog/pytorch/pytorch%20lightning/loss%20function/gpu/2021/03/07/BLIP.html +def contrastive_loss(logits: torch.Tensor) -> torch.Tensor: + return nn.functional.cross_entropy(logits, torch.arange(len(logits), device=logits.device)) + + +# Copied from transformers.models.clip.modeling_clip.clip_loss with clip->blip +def blip_loss(similarity: torch.Tensor) -> torch.Tensor: + caption_loss = contrastive_loss(similarity) + image_loss = contrastive_loss(similarity.t()) + return (caption_loss + image_loss) / 2.0 + + +@dataclass +# Copied from transformers.models.clip.modeling_clip.CLIPVisionModelOutput with CLIP->BLIP +class BLIPVisionModelOutput(ModelOutput): + """ + Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states. + + Args: + image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`): + The image embeddings obtained by applying the projection layer to the pooler_output. + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. + + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + """ + + image_embeds: Optional[torch.FloatTensor] = None + last_hidden_state: torch.FloatTensor = None + hidden_states: Optional[Tuple[torch.FloatTensor]] = None + attentions: Optional[Tuple[torch.FloatTensor]] = None + + +@dataclass +# Copied from transformers.models.clip.modeling_clip.CLIPTextModelOutput with CLIP->BLIP +class BLIPTextModelOutput(ModelOutput): + """ + Base class for text model's outputs that also contains a pooling of the last hidden states. + + Args: + text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`): + The text embeddings obtained by applying the projection layer to the pooler_output. + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. + + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + """ + + text_embeds: Optional[torch.FloatTensor] = None + last_hidden_state: torch.FloatTensor = None + hidden_states: Optional[Tuple[torch.FloatTensor]] = None + attentions: Optional[Tuple[torch.FloatTensor]] = None + + +@dataclass +# Copied from transformers.models.clip.modeling_clip.CLIPOutput with CLIP->BLIP +class BLIPOutput(ModelOutput): + """ + Args: + loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`): + Contrastive loss for image-text similarity. + logits_per_image:(`torch.FloatTensor` of shape `(image_batch_size, text_batch_size)`): + The scaled dot product scores between `image_embeds` and `text_embeds`. This represents the image-text + similarity scores. + logits_per_text:(`torch.FloatTensor` of shape `(text_batch_size, image_batch_size)`): + The scaled dot product scores between `text_embeds` and `image_embeds`. This represents the text-image + similarity scores. + text_embeds(`torch.FloatTensor` of shape `(batch_size, output_dim`): + The text embeddings obtained by applying the projection layer to the pooled output of [`BLIPTextModel`]. + image_embeds(`torch.FloatTensor` of shape `(batch_size, output_dim`): + The image embeddings obtained by applying the projection layer to the pooled output of [`BLIPVisionModel`]. + text_model_output(`BaseModelOutputWithPooling`): + The output of the [`BLIPTextModel`]. + vision_model_output(`BaseModelOutputWithPooling`): + The output of the [`BLIPVisionModel`]. + """ + + loss: Optional[torch.FloatTensor] = None + logits_per_image: torch.FloatTensor = None + logits_per_text: torch.FloatTensor = None + text_embeds: torch.FloatTensor = None + image_embeds: torch.FloatTensor = None + text_model_output: BaseModelOutputWithPooling = None + vision_model_output: BaseModelOutputWithPooling = None + + def to_tuple(self) -> Tuple[Any]: + return tuple( + self[k] if k not in ["text_model_output", "vision_model_output"] else getattr(self, k).to_tuple() + for k in self.keys() + ) + + +# Copied from transformers.models.clip.modeling_clip.CLIPVisionEmbeddings with CLIP->BLIP +class BLIPVisionEmbeddings(nn.Module): + def __init__(self, config: BLIPVisionConfig): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.image_size = config.image_size + self.patch_size = config.patch_size + + self.class_embedding = nn.Parameter(torch.randn(self.embed_dim)) + + self.patch_embedding = nn.Conv2d( + in_channels=3, out_channels=self.embed_dim, kernel_size=self.patch_size, stride=self.patch_size, bias=False + ) + + self.num_patches = (self.image_size // self.patch_size) ** 2 + self.num_positions = self.num_patches + 1 + self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim) + self.register_buffer("position_ids", torch.arange(self.num_positions).expand((1, -1))) + + def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: + batch_size = pixel_values.shape[0] + patch_embeds = self.patch_embedding(pixel_values) # shape = [*, width, grid, grid] + patch_embeds = patch_embeds.flatten(2).transpose(1, 2) + + class_embeds = self.class_embedding.expand(batch_size, 1, -1) + embeddings = torch.cat([class_embeds, patch_embeds], dim=1) + embeddings = embeddings + self.position_embedding(self.position_ids) + return embeddings + + +# Copied from transformers.models.clip.modeling_clip.CLIPTextEmbeddings with CLIP->BLIP +class BLIPTextEmbeddings(nn.Module): + def __init__(self, config: BLIPTextConfig): + super().__init__() + embed_dim = config.hidden_size + + self.token_embedding = nn.Embedding(config.vocab_size, embed_dim) + self.position_embedding = nn.Embedding(config.max_position_embeddings, embed_dim) + + # position_ids (1, len position emb) is contiguous in memory and exported when serialized + self.register_buffer("position_ids", torch.arange(config.max_position_embeddings).expand((1, -1))) + + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + ) -> torch.Tensor: + seq_length = input_ids.shape[-1] if input_ids is not None else inputs_embeds.shape[-2] + + if position_ids is None: + position_ids = self.position_ids[:, :seq_length] + + if inputs_embeds is None: + inputs_embeds = self.token_embedding(input_ids) + + position_embeddings = self.position_embedding(position_ids) + embeddings = inputs_embeds + position_embeddings + + return embeddings + + +# Copied from transformers.models.clip.modeling_clip.CLIPAttention with CLIP->BLIP +class BLIPAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__(self, config): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.num_heads = config.num_attention_heads + self.head_dim = self.embed_dim // self.num_heads + if self.head_dim * self.num_heads != self.embed_dim: + raise ValueError( + f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:" + f" {self.num_heads})." + ) + self.scale = self.head_dim**-0.5 + self.dropout = config.attention_dropout + + self.k_proj = nn.Linear(self.embed_dim, self.embed_dim) + self.v_proj = nn.Linear(self.embed_dim, self.embed_dim) + self.q_proj = nn.Linear(self.embed_dim, self.embed_dim) + self.out_proj = nn.Linear(self.embed_dim, self.embed_dim) + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + causal_attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + """Input shape: Batch x Time x Channel""" + + bsz, tgt_len, embed_dim = hidden_states.size() + + # get query proj + query_states = self.q_proj(hidden_states) * self.scale + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + + proj_shape = (bsz * self.num_heads, -1, self.head_dim) + query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) + key_states = key_states.view(*proj_shape) + value_states = value_states.view(*proj_shape) + + src_len = key_states.size(1) + attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + + if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): + raise ValueError( + f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is" + f" {attn_weights.size()}" + ) + + # apply the causal_attention_mask first + if causal_attention_mask is not None: + if causal_attention_mask.size() != (bsz, 1, tgt_len, src_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is" + f" {causal_attention_mask.size()}" + ) + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + causal_attention_mask + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, tgt_len, src_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + attn_weights = nn.functional.softmax(attn_weights, dim=-1) + + if output_attentions: + # this operation is a bit akward, but it's required to + # make sure that attn_weights keeps its gradient. + # In order to do so, attn_weights have to reshaped + # twice and have to be reused in the following + attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) + else: + attn_weights_reshaped = None + + attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + + attn_output = torch.bmm(attn_probs, value_states) + + if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim) + attn_output = attn_output.transpose(1, 2) + attn_output = attn_output.reshape(bsz, tgt_len, embed_dim) + + attn_output = self.out_proj(attn_output) + + return attn_output, attn_weights_reshaped + + +# Copied from transformers.models.clip.modeling_clip.CLIPMLP with CLIP->BLIP +class BLIPMLP(nn.Module): + def __init__(self, config): + super().__init__() + self.config = config + self.activation_fn = ACT2FN[config.hidden_act] + self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size) + self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.fc1(hidden_states) + hidden_states = self.activation_fn(hidden_states) + hidden_states = self.fc2(hidden_states) + return hidden_states + + +# Copied from transformers.models.clip.modeling_clip.CLIPEncoderLayer with CLIP->BLIP +class BLIPEncoderLayer(nn.Module): + def __init__(self, config: BLIPConfig): + super().__init__() + self.embed_dim = config.hidden_size + self.self_attn = BLIPAttention(config) + self.layer_norm1 = nn.LayerNorm(self.embed_dim) + self.mlp = BLIPMLP(config) + self.layer_norm2 = nn.LayerNorm(self.embed_dim) + + def forward( + self, + hidden_states: torch.Tensor, + attention_mask: torch.Tensor, + causal_attention_mask: torch.Tensor, + output_attentions: Optional[bool] = False, + ) -> Tuple[torch.FloatTensor]: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` + attention_mask (`torch.FloatTensor`): attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + `(config.encoder_attention_heads,)`. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + """ + residual = hidden_states + + hidden_states = self.layer_norm1(hidden_states) + hidden_states, attn_weights = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + causal_attention_mask=causal_attention_mask, + output_attentions=output_attentions, + ) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.layer_norm2(hidden_states) + hidden_states = self.mlp(hidden_states) + hidden_states = residual + hidden_states + + outputs = (hidden_states,) + + if output_attentions: + outputs += (attn_weights,) + + return outputs + + +# Copied from transformers.models.clip.modeling_clip.CLIPPreTrainedModel with CLIP->BLIP,clip->blip +class BLIPPreTrainedModel(PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = BLIPConfig + base_model_prefix = "blip" + supports_gradient_checkpointing = True + _keys_to_ignore_on_load_missing = [r"position_ids"] + + def _init_weights(self, module): + """Initialize the weights""" + factor = self.config.initializer_factor + if isinstance(module, BLIPTextEmbeddings): + module.token_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02) + module.position_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02) + elif isinstance(module, BLIPVisionEmbeddings): + factor = self.config.initializer_factor + nn.init.normal_(module.class_embedding, mean=0.0, std=module.embed_dim**-0.5 * factor) + nn.init.normal_(module.patch_embedding.weight, std=module.config.initializer_range * factor) + nn.init.normal_(module.position_embedding.weight, std=module.config.initializer_range * factor) + elif isinstance(module, BLIPAttention): + factor = self.config.initializer_factor + in_proj_std = (module.embed_dim**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor + out_proj_std = (module.embed_dim**-0.5) * factor + nn.init.normal_(module.q_proj.weight, std=in_proj_std) + nn.init.normal_(module.k_proj.weight, std=in_proj_std) + nn.init.normal_(module.v_proj.weight, std=in_proj_std) + nn.init.normal_(module.out_proj.weight, std=out_proj_std) + elif isinstance(module, BLIPMLP): + factor = self.config.initializer_factor + in_proj_std = ( + (module.config.hidden_size**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor + ) + fc_std = (2 * module.config.hidden_size) ** -0.5 * factor + nn.init.normal_(module.fc1.weight, std=fc_std) + nn.init.normal_(module.fc2.weight, std=in_proj_std) + elif isinstance(module, BLIPModel): + nn.init.normal_( + module.text_projection.weight, + std=module.text_embed_dim**-0.5 * self.config.initializer_factor, + ) + nn.init.normal_( + module.visual_projection.weight, + std=module.vision_embed_dim**-0.5 * self.config.initializer_factor, + ) + elif isinstance(module, BLIPVisionModelWithProjection): + nn.init.normal_( + module.visual_projection.weight, + std=self.config.hidden_size**-0.5 * self.config.initializer_factor, + ) + elif isinstance(module, BLIPTextModelWithProjection): + nn.init.normal_( + module.text_projection.weight, + std=self.config.hidden_size**-0.5 * self.config.initializer_factor, + ) + + if isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, BLIPEncoder): + module.gradient_checkpointing = value + + +BLIP_START_DOCSTRING = r""" + This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the + library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads + etc.) + + This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. + Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage + and behavior. + + Parameters: + config ([`BLIPConfig`]): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the + configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. +""" + +BLIP_TEXT_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + + Indices can be obtained using [`CLIPTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + +BLIP_VISION_INPUTS_DOCSTRING = r""" + Args: + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using + [`CLIPFeatureExtractor`]. See [`CLIPFeatureExtractor.__call__`] for details. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + +BLIP_INPUTS_DOCSTRING = r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide + it. + + Indices can be obtained using [`CLIPTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): + Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, + config.max_position_embeddings - 1]`. + + [What are position IDs?](../glossary#position-ids) + pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using + [`CLIPFeatureExtractor`]. See [`CLIPFeatureExtractor.__call__`] for details. + return_loss (`bool`, *optional*): + Whether or not to return the contrastive loss. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned + tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for + more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. +""" + + +# Copied from transformers.models.clip.modeling_clip.CLIPEncoder with CLIP->BLIP +class BLIPEncoder(nn.Module): + """ + Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a + [`BLIPEncoderLayer`]. + + Args: + config: BLIPConfig + """ + + def __init__(self, config: BLIPConfig): + super().__init__() + self.config = config + self.layers = nn.ModuleList([BLIPEncoderLayer(config) for _ in range(config.num_hidden_layers)]) + self.gradient_checkpointing = False + + def forward( + self, + inputs_embeds, + attention_mask: Optional[torch.Tensor] = None, + causal_attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutput]: + r""" + Args: + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + causal_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Causal mask for the text model. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + encoder_states = () if output_hidden_states else None + all_attentions = () if output_attentions else None + + hidden_states = inputs_embeds + for idx, encoder_layer in enumerate(self.layers): + if output_hidden_states: + encoder_states = encoder_states + (hidden_states,) + if self.gradient_checkpointing and self.training: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs, output_attentions) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(encoder_layer), + hidden_states, + attention_mask, + causal_attention_mask, + ) + else: + layer_outputs = encoder_layer( + hidden_states, + attention_mask, + causal_attention_mask, + output_attentions=output_attentions, + ) + + hidden_states = layer_outputs[0] + + if output_attentions: + all_attentions = all_attentions + (layer_outputs[1],) + + if output_hidden_states: + encoder_states = encoder_states + (hidden_states,) + + if not return_dict: + return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None) + return BaseModelOutput( + last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions + ) + + +class BLIPTextTransformer(nn.Module): + def __init__(self, config: BLIPTextConfig): + super().__init__() + self.config = config + embed_dim = config.hidden_size + self.embeddings = BLIPTextEmbeddings(config) + self.encoder = BLIPEncoder(config) + self.final_layer_norm = nn.LayerNorm(embed_dim) + + @add_start_docstrings_to_model_forward(BLIP_TEXT_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=BLIPTextConfig) + def forward( + self, + input_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPooling]: + r""" + Returns: + + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if input_ids is None: + raise ValueError("You have to specify either input_ids") + + input_shape = input_ids.size() + input_ids = input_ids.view(-1, input_shape[-1]) + + hidden_states = self.embeddings(input_ids=input_ids, position_ids=position_ids) + + bsz, seq_len = input_shape + # BLIP's text model uses causal mask, prepare it here. + # https://github.com/openai/BLIP/blob/cfcffb90e69f37bf2ff1e988237a0fbe41f33c04/blip/model.py#L324 + causal_attention_mask = self._build_causal_attention_mask(bsz, seq_len, hidden_states.dtype).to( + hidden_states.device + ) + # expand attention_mask + if attention_mask is not None: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + attention_mask = _expand_mask(attention_mask, hidden_states.dtype) + + encoder_outputs = self.encoder( + inputs_embeds=hidden_states, + attention_mask=attention_mask, + causal_attention_mask=causal_attention_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + last_hidden_state = encoder_outputs[0] + last_hidden_state = self.final_layer_norm(last_hidden_state) + + # text_embeds.shape = [batch_size, sequence_length, transformer.width] + # take features from the eot embedding (eot_token is the highest number in each sequence) + # casting to torch.int for onnx compatibility: argmax doesn't support int64 inputs with opset 14 + pooled_output = last_hidden_state[ + torch.arange(last_hidden_state.shape[0], device=input_ids.device), input_ids.to(torch.int).argmax(dim=-1) + ] + + if not return_dict: + return (last_hidden_state, pooled_output) + encoder_outputs[1:] + + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=pooled_output, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + ) + + def _build_causal_attention_mask(self, bsz, seq_len, dtype): + # lazily create causal attention mask, with full attention between the vision tokens + # pytorch uses additive attention mask; fill with -inf + mask = torch.empty(bsz, seq_len, seq_len, dtype=dtype) + mask.fill_(torch.tensor(torch.finfo(dtype).min)) + mask.triu_(1) # zero out the lower diagonal + mask = mask.unsqueeze(1) # expand mask + return mask + + +@add_start_docstrings( + """The text model from BLIP without any head or projection on top.""", + BLIP_START_DOCSTRING, +) +class BLIPTextModel(BLIPPreTrainedModel): + config_class = BLIPTextConfig + + _no_split_modules = ["BLIPEncoderLayer"] + + def __init__(self, config: BLIPTextConfig): + super().__init__(config) + self.text_model = BLIPTextTransformer(config) + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self) -> nn.Module: + return self.text_model.embeddings.token_embedding + + def set_input_embeddings(self, value): + self.text_model.embeddings.token_embedding = value + + @add_start_docstrings_to_model_forward(BLIP_TEXT_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=BLIPTextConfig) + def forward( + self, + input_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPooling]: + r""" + Returns: + + Examples: + + ```python + >>> from transformers import CLIPTokenizer, BLIPTextModel + + >>> model = BLIPTextModel.from_pretrained("ybelkada/blip-base") + >>> tokenizer = CLIPTokenizer.from_pretrained("ybelkada/blip-base") + + >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") + + >>> outputs = model(**inputs) + >>> last_hidden_state = outputs.last_hidden_state + >>> pooled_output = outputs.pooler_output # pooled (EOS token) states + ```""" + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + return self.text_model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + +class BLIPVisionTransformer(nn.Module): + def __init__(self, config: BLIPVisionConfig): + super().__init__() + self.config = config + embed_dim = config.hidden_size + + self.embeddings = BLIPVisionEmbeddings(config) + self.pre_layrnorm = nn.LayerNorm(embed_dim) + self.encoder = BLIPEncoder(config) + self.post_layernorm = nn.LayerNorm(embed_dim) + + @add_start_docstrings_to_model_forward(BLIP_VISION_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=BLIPVisionConfig) + def forward( + self, + pixel_values: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPooling]: + r""" + Returns: + + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if pixel_values is None: + raise ValueError("You have to specify pixel_values") + + hidden_states = self.embeddings(pixel_values) + hidden_states = self.pre_layrnorm(hidden_states) + + encoder_outputs = self.encoder( + inputs_embeds=hidden_states, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + last_hidden_state = encoder_outputs[0] + pooled_output = last_hidden_state[:, 0, :] + pooled_output = self.post_layernorm(pooled_output) + + if not return_dict: + return (last_hidden_state, pooled_output) + encoder_outputs[1:] + + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=pooled_output, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + ) + + +@add_start_docstrings( + """The vision model from BLIP without any head or projection on top.""", + BLIP_START_DOCSTRING, +) +class BLIPVisionModel(BLIPPreTrainedModel): + config_class = BLIPVisionConfig + main_input_name = "pixel_values" + + def __init__(self, config: BLIPVisionConfig): + super().__init__(config) + self.vision_model = BLIPVisionTransformer(config) + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self) -> nn.Module: + return self.vision_model.embeddings.patch_embedding + + @add_start_docstrings_to_model_forward(BLIP_VISION_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=BLIPVisionConfig) + def forward( + self, + pixel_values: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPooling]: + r""" + Returns: + + Examples: + + ```python + >>> from PIL import Image + >>> import requests + >>> from transformers import CLIPProcessor, BLIPVisionModel + + >>> model = BLIPVisionModel.from_pretrained("ybelkada/blip-base") + >>> processor = CLIPProcessor.from_pretrained("ybelkada/blip-base") + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> inputs = processor(images=image, return_tensors="pt") + + >>> outputs = model(**inputs) + >>> last_hidden_state = outputs.last_hidden_state + >>> pooled_output = outputs.pooler_output # pooled CLS states + ```""" + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + return self.vision_model( + pixel_values=pixel_values, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + +@add_start_docstrings(BLIP_START_DOCSTRING) +class BLIPModel(BLIPPreTrainedModel): + config_class = BLIPConfig + + def __init__(self, config: BLIPConfig): + super().__init__(config) + + if not isinstance(config.text_config, BLIPTextConfig): + raise ValueError( + "config.text_config is expected to be of type BLIPTextConfig but is of type" + f" {type(config.text_config)}." + ) + + if not isinstance(config.vision_config, BLIPVisionConfig): + raise ValueError( + "config.vision_config is expected to be of type BLIPVisionConfig but is of type" + f" {type(config.vision_config)}." + ) + + text_config = config.text_config + vision_config = config.vision_config + + self.projection_dim = config.projection_dim + self.text_embed_dim = text_config.hidden_size + self.vision_embed_dim = vision_config.hidden_size + + self.text_model = BLIPTextTransformer(text_config) + self.vision_model = BLIPVisionTransformer(vision_config) + + self.visual_projection = nn.Linear(self.vision_embed_dim, self.projection_dim, bias=False) + self.text_projection = nn.Linear(self.text_embed_dim, self.projection_dim, bias=False) + self.logit_scale = nn.Parameter(torch.ones([]) * self.config.logit_scale_init_value) + + # Initialize weights and apply final processing + self.post_init() + + @add_start_docstrings_to_model_forward(BLIP_TEXT_INPUTS_DOCSTRING) + def get_text_features( + self, + input_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> torch.FloatTensor: + r""" + Returns: + text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by + applying the projection layer to the pooled output of [`BLIPTextModel`]. + + Examples: + + ```python + >>> from transformers import CLIPTokenizer, BLIPModel + + >>> model = BLIPModel.from_pretrained("ybelkada/blip-base") + >>> tokenizer = CLIPTokenizer.from_pretrained("ybelkada/blip-base") + + >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") + >>> text_features = model.get_text_features(**inputs) + ```""" + # Use BLIP model's config for some fields (if specified) instead of those of vision & text components. + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + text_outputs = self.text_model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + pooled_output = text_outputs[1] + text_features = self.text_projection(pooled_output) + + return text_features + + @add_start_docstrings_to_model_forward(BLIP_VISION_INPUTS_DOCSTRING) + def get_image_features( + self, + pixel_values: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> torch.FloatTensor: + r""" + Returns: + image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by + applying the projection layer to the pooled output of [`BLIPVisionModel`]. + + Examples: + + ```python + >>> from PIL import Image + >>> import requests + >>> from transformers import CLIPProcessor, BLIPModel + + >>> model = BLIPModel.from_pretrained("ybelkada/blip-base") + >>> processor = CLIPProcessor.from_pretrained("ybelkada/blip-base") + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> inputs = processor(images=image, return_tensors="pt") + + >>> image_features = model.get_image_features(**inputs) + ```""" + # Use BLIP model's config for some fields (if specified) instead of those of vision & text components. + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + vision_outputs = self.vision_model( + pixel_values=pixel_values, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + pooled_output = vision_outputs[1] # pooled_output + image_features = self.visual_projection(pooled_output) + + return image_features + + @add_start_docstrings_to_model_forward(BLIP_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=BLIPOutput, config_class=BLIPConfig) + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + pixel_values: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + return_loss: Optional[bool] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BLIPOutput]: + r""" + Returns: + + Examples: + + ```python + >>> from PIL import Image + >>> import requests + >>> from transformers import CLIPProcessor, BLIPModel + + >>> model = BLIPModel.from_pretrained("ybelkada/blip-base") + >>> processor = CLIPProcessor.from_pretrained("ybelkada/blip-base") + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> inputs = processor( + ... text=["a photo of a cat", "a photo of a dog"], images=image, return_tensors="pt", padding=True + ... ) + + >>> outputs = model(**inputs) + >>> logits_per_image = outputs.logits_per_image # this is the image-text similarity score + >>> probs = logits_per_image.softmax(dim=1) # we can take the softmax to get the label probabilities + ```""" + # Use BLIP model's config for some fields (if specified) instead of those of vision & text components. + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + vision_outputs = self.vision_model( + pixel_values=pixel_values, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + text_outputs = self.text_model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + image_embeds = vision_outputs[1] + image_embeds = self.visual_projection(image_embeds) + + text_embeds = text_outputs[1] + text_embeds = self.text_projection(text_embeds) + + # normalized features + image_embeds = image_embeds / image_embeds.norm(p=2, dim=-1, keepdim=True) + text_embeds = text_embeds / text_embeds.norm(p=2, dim=-1, keepdim=True) + + # cosine similarity as logits + logit_scale = self.logit_scale.exp() + logits_per_text = torch.matmul(text_embeds, image_embeds.t()) * logit_scale + logits_per_image = logits_per_text.t() + + loss = None + if return_loss: + loss = blip_loss(logits_per_text) + + if not return_dict: + output = (logits_per_image, logits_per_text, text_embeds, image_embeds, text_outputs, vision_outputs) + return ((loss,) + output) if loss is not None else output + + return BLIPOutput( + loss=loss, + logits_per_image=logits_per_image, + logits_per_text=logits_per_text, + text_embeds=text_embeds, + image_embeds=image_embeds, + text_model_output=text_outputs, + vision_model_output=vision_outputs, + ) + + +@add_start_docstrings( + """ + BLIP Text Model with a projection layer on top (a linear layer on top of the pooled output). + """, + BLIP_START_DOCSTRING, +) +class BLIPTextModelWithProjection(BLIPPreTrainedModel): + config_class = BLIPTextConfig + + _no_split_modules = ["BLIPEncoderLayer"] + + def __init__(self, config: BLIPTextConfig): + super().__init__(config) + + self.text_model = BLIPTextTransformer(config) + + self.text_projection = nn.Linear(config.hidden_size, config.projection_dim, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self) -> nn.Module: + return self.text_model.embeddings.token_embedding + + def set_input_embeddings(self, value): + self.text_model.embeddings.token_embedding = value + + @add_start_docstrings_to_model_forward(BLIP_TEXT_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=BLIPTextModelOutput, config_class=BLIPTextConfig) + def forward( + self, + input_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BLIPTextModelOutput]: + r""" + Returns: + + Examples: + + ```python + >>> from transformers import CLIPTokenizer, BLIPTextModelWithProjection + + >>> model = BLIPTextModelWithProjection.from_pretrained("ybelkada/blip-base") + >>> tokenizer = CLIPTokenizer.from_pretrained("ybelkada/blip-base") + + >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") + + >>> outputs = model(**inputs) + >>> text_embeds = outputs.text_embeds + ```""" + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + text_outputs = self.text_model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + pooled_output = text_outputs[1] + + text_embeds = self.text_projection(pooled_output) + + if not return_dict: + outputs = (text_embeds, text_outputs[0]) + text_outputs[2:] + return tuple(output for output in outputs if output is not None) + + return BLIPTextModelOutput( + text_embeds=text_embeds, + last_hidden_state=text_outputs.last_hidden_state, + hidden_states=text_outputs.hidden_states, + attentions=text_outputs.attentions, + ) + + +@add_start_docstrings( + """ + BLIP Vision Model with a projection layer on top (a linear layer on top of the pooled output). + """, + BLIP_START_DOCSTRING, +) +class BLIPVisionModelWithProjection(BLIPPreTrainedModel): + config_class = BLIPVisionConfig + main_input_name = "pixel_values" + + def __init__(self, config: BLIPVisionConfig): + super().__init__(config) + + self.vision_model = BLIPVisionTransformer(config) + + self.visual_projection = nn.Linear(config.hidden_size, config.projection_dim, bias=False) + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self) -> nn.Module: + return self.vision_model.embeddings.patch_embedding + + @add_start_docstrings_to_model_forward(BLIP_VISION_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=BLIPVisionModelOutput, config_class=BLIPVisionConfig) + def forward( + self, + pixel_values: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BLIPVisionModelOutput]: + r""" + Returns: + + Examples: + + ```python + >>> from PIL import Image + >>> import requests + >>> from transformers import CLIPProcessor, BLIPVisionModelWithProjection + + >>> model = BLIPVisionModelWithProjection.from_pretrained("ybelkada/blip-base") + >>> processor = CLIPProcessor.from_pretrained("ybelkada/blip-base") + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> inputs = processor(images=image, return_tensors="pt") + + >>> outputs = model(**inputs) + >>> image_embeds = outputs.image_embeds + ```""" + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + vision_outputs = self.vision_model( + pixel_values=pixel_values, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + pooled_output = vision_outputs[1] # pooled_output + + image_embeds = self.visual_projection(pooled_output) + + if not return_dict: + outputs = (image_embeds, vision_outputs[0]) + vision_outputs[2:] + return tuple(output for output in outputs if output is not None) + + return BLIPVisionModelOutput( + image_embeds=image_embeds, + last_hidden_state=vision_outputs.last_hidden_state, + hidden_states=vision_outputs.hidden_states, + attentions=vision_outputs.attentions, + ) diff --git a/tests/models/blip/__init__.py b/tests/models/blip/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/models/blip/test_modeling_blip.py b/tests/models/blip/test_modeling_blip.py new file mode 100644 index 000000000000..38bc771d8f06 --- /dev/null +++ b/tests/models/blip/test_modeling_blip.py @@ -0,0 +1,737 @@ +# coding=utf-8 +# Copyright 2022 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Testing suite for the PyTorch BLIP model. """ + + +import inspect +import os +import tempfile +import unittest + +import numpy as np + +import requests +import transformers +from transformers import BLIPConfig, BLIPTextConfig, BLIPVisionConfig +from transformers.testing_utils import ( + is_flax_available, + is_pt_flax_cross_test, + require_torch, + require_vision, + slow, + torch_device, +) +from transformers.utils import is_torch_available, is_vision_available + +from ...test_configuration_common import ConfigTester +from ...test_modeling_common import ( + ModelTesterMixin, + _config_zero_init, + floats_tensor, + ids_tensor, + random_attention_mask, +) + + +if is_torch_available(): + import torch + from torch import nn + + from transformers import ( + BLIPModel, + BLIPTextModel, + BLIPTextModelWithProjection, + BLIPVisionModel, + BLIPVisionModelWithProjection, + ) + from transformers.models.blip.modeling_blip import BLIP_PRETRAINED_MODEL_ARCHIVE_LIST + + +if is_vision_available(): + from PIL import Image + + from transformers import CLIPProcessor + + +if is_flax_available(): + import jax.numpy as jnp + from transformers.modeling_flax_pytorch_utils import ( + convert_pytorch_state_dict_to_flax, + load_flax_weights_in_pytorch_model, + ) + + +class BLIPVisionModelTester: + def __init__( + self, + parent, + batch_size=12, + image_size=30, + patch_size=2, + num_channels=3, + is_training=True, + hidden_size=32, + projection_dim=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + dropout=0.1, + attention_dropout=0.1, + initializer_range=0.02, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.image_size = image_size + self.patch_size = patch_size + self.num_channels = num_channels + self.is_training = is_training + self.hidden_size = hidden_size + self.projection_dim = projection_dim + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.dropout = dropout + self.attention_dropout = attention_dropout + self.initializer_range = initializer_range + self.scope = scope + + # in ViT, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token) + num_patches = (image_size // patch_size) ** 2 + self.seq_length = num_patches + 1 + + def prepare_config_and_inputs(self): + pixel_values = floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]) + config = self.get_config() + + return config, pixel_values + + def get_config(self): + return BLIPVisionConfig( + image_size=self.image_size, + patch_size=self.patch_size, + num_channels=self.num_channels, + hidden_size=self.hidden_size, + projection_dim=self.projection_dim, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + intermediate_size=self.intermediate_size, + dropout=self.dropout, + attention_dropout=self.attention_dropout, + initializer_range=self.initializer_range, + ) + + def create_and_check_model(self, config, pixel_values): + model = BLIPVisionModel(config=config) + model.to(torch_device) + model.eval() + with torch.no_grad(): + result = model(pixel_values) + # expected sequence length = num_patches + 1 (we add 1 for the [CLS] token) + image_size = (self.image_size, self.image_size) + patch_size = (self.patch_size, self.patch_size) + num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0]) + self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, num_patches + 1, self.hidden_size)) + self.parent.assertEqual(result.pooler_output.shape, (self.batch_size, self.hidden_size)) + + def create_and_check_model_with_projection(self, config, pixel_values): + model = BLIPVisionModelWithProjection(config=config) + model.to(torch_device) + model.eval() + with torch.no_grad(): + result = model(pixel_values) + # expected sequence length = num_patches + 1 (we add 1 for the [CLS] token) + image_size = (self.image_size, self.image_size) + patch_size = (self.patch_size, self.patch_size) + num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0]) + self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, num_patches + 1, self.hidden_size)) + self.parent.assertEqual(result.image_embeds.shape, (self.batch_size, self.projection_dim)) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + config, pixel_values = config_and_inputs + inputs_dict = {"pixel_values": pixel_values} + return config, inputs_dict + + +@require_torch +class BLIPVisionModelTest(ModelTesterMixin, unittest.TestCase): + """ + Here we also overwrite some of the tests of test_modeling_common.py, as BLIP does not use input_ids, inputs_embeds, + attention_mask and seq_length. + """ + + all_model_classes = (BLIPVisionModel, BLIPVisionModelWithProjection) if is_torch_available() else () + fx_compatible = False + test_pruning = False + test_resize_embeddings = False + test_head_masking = False + + def setUp(self): + self.model_tester = BLIPVisionModelTester(self) + self.config_tester = ConfigTester(self, config_class=BLIPVisionConfig, has_text_modality=False, hidden_size=37) + + def test_config(self): + self.config_tester.run_common_tests() + + @unittest.skip(reason="BLIP does not use inputs_embeds") + def test_inputs_embeds(self): + pass + + def test_model_common_attributes(self): + config, _ = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + self.assertIsInstance(model.get_input_embeddings(), (nn.Module)) + x = model.get_output_embeddings() + self.assertTrue(x is None or isinstance(x, nn.Linear)) + + def test_forward_signature(self): + config, _ = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + signature = inspect.signature(model.forward) + # signature.parameters is an OrderedDict => so arg_names order is deterministic + arg_names = [*signature.parameters.keys()] + + expected_arg_names = ["pixel_values"] + self.assertListEqual(arg_names[:1], expected_arg_names) + + def test_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_model(*config_and_inputs) + + def test_model_with_projection(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_model_with_projection(*config_and_inputs) + + def test_training(self): + pass + + def test_training_gradient_checkpointing(self): + pass + + @unittest.skip(reason="BLIPVisionModel has no base class and is not available in MODEL_MAPPING") + def test_save_load_fast_init_from_base(self): + pass + + @unittest.skip(reason="BLIPVisionModel has no base class and is not available in MODEL_MAPPING") + def test_save_load_fast_init_to_base(self): + pass + + @slow + def test_model_from_pretrained(self): + for model_name in BLIP_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: + model = BLIPVisionModel.from_pretrained(model_name) + self.assertIsNotNone(model) + + @slow + def test_model_with_projection_from_pretrained(self): + for model_name in BLIP_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: + model = BLIPVisionModelWithProjection.from_pretrained(model_name) + self.assertIsNotNone(model) + self.assertTrue(hasattr(model, "visual_projection")) + + +class BLIPTextModelTester: + def __init__( + self, + parent, + batch_size=12, + seq_length=7, + is_training=True, + use_input_mask=True, + use_labels=True, + vocab_size=99, + hidden_size=32, + projection_dim=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + dropout=0.1, + attention_dropout=0.1, + max_position_embeddings=512, + initializer_range=0.02, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_mask = use_input_mask + self.use_labels = use_labels + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.projection_dim = projection_dim + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.dropout = dropout + self.attention_dropout = attention_dropout + self.max_position_embeddings = max_position_embeddings + self.initializer_range = initializer_range + self.scope = scope + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = random_attention_mask([self.batch_size, self.seq_length]) + + if input_mask is not None: + batch_size, seq_length = input_mask.shape + rnd_start_indices = np.random.randint(1, seq_length - 1, size=(batch_size,)) + for batch_idx, start_index in enumerate(rnd_start_indices): + input_mask[batch_idx, :start_index] = 1 + input_mask[batch_idx, start_index:] = 0 + + config = self.get_config() + + return config, input_ids, input_mask + + def get_config(self): + return BLIPTextConfig( + vocab_size=self.vocab_size, + hidden_size=self.hidden_size, + projection_dim=self.projection_dim, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + intermediate_size=self.intermediate_size, + dropout=self.dropout, + attention_dropout=self.attention_dropout, + max_position_embeddings=self.max_position_embeddings, + initializer_range=self.initializer_range, + ) + + def create_and_check_model(self, config, input_ids, input_mask): + model = BLIPTextModel(config=config) + model.to(torch_device) + model.eval() + with torch.no_grad(): + result = model(input_ids, attention_mask=input_mask) + result = model(input_ids) + self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size)) + self.parent.assertEqual(result.pooler_output.shape, (self.batch_size, self.hidden_size)) + + def create_and_check_model_with_projection(self, config, input_ids, input_mask): + model = BLIPTextModelWithProjection(config=config) + model.to(torch_device) + model.eval() + with torch.no_grad(): + result = model(input_ids, attention_mask=input_mask) + result = model(input_ids) + self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size)) + self.parent.assertEqual(result.text_embeds.shape, (self.batch_size, self.projection_dim)) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + config, input_ids, input_mask = config_and_inputs + inputs_dict = {"input_ids": input_ids, "attention_mask": input_mask} + return config, inputs_dict + + +@require_torch +class BLIPTextModelTest(ModelTesterMixin, unittest.TestCase): + + all_model_classes = (BLIPTextModel, BLIPTextModelWithProjection) if is_torch_available() else () + fx_compatible = False + test_pruning = False + test_head_masking = False + + def setUp(self): + self.model_tester = BLIPTextModelTester(self) + self.config_tester = ConfigTester(self, config_class=BLIPTextConfig, hidden_size=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_model(*config_and_inputs) + + def test_model_with_projection(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_model_with_projection(*config_and_inputs) + + def test_training(self): + pass + + def test_training_gradient_checkpointing(self): + pass + + @unittest.skip(reason="BLIP does not use inputs_embeds") + def test_inputs_embeds(self): + pass + + @unittest.skip(reason="BLIPTextModel has no base class and is not available in MODEL_MAPPING") + def test_save_load_fast_init_from_base(self): + pass + + @unittest.skip(reason="BLIPTextModel has no base class and is not available in MODEL_MAPPING") + def test_save_load_fast_init_to_base(self): + pass + + @slow + def test_model_from_pretrained(self): + for model_name in BLIP_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: + model = BLIPTextModel.from_pretrained(model_name) + self.assertIsNotNone(model) + + @slow + def test_model_with_projection_from_pretrained(self): + for model_name in BLIP_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: + model = BLIPTextModelWithProjection.from_pretrained(model_name) + self.assertIsNotNone(model) + self.assertTrue(hasattr(model, "text_projection")) + + +class BLIPModelTester: + def __init__(self, parent, text_kwargs=None, vision_kwargs=None, is_training=True): + + if text_kwargs is None: + text_kwargs = {} + if vision_kwargs is None: + vision_kwargs = {} + + self.parent = parent + self.text_model_tester = BLIPTextModelTester(parent, **text_kwargs) + self.vision_model_tester = BLIPVisionModelTester(parent, **vision_kwargs) + self.is_training = is_training + + def prepare_config_and_inputs(self): + text_config, input_ids, attention_mask = self.text_model_tester.prepare_config_and_inputs() + vision_config, pixel_values = self.vision_model_tester.prepare_config_and_inputs() + + config = self.get_config() + + return config, input_ids, attention_mask, pixel_values + + def get_config(self): + return BLIPConfig.from_text_vision_configs( + self.text_model_tester.get_config(), self.vision_model_tester.get_config(), projection_dim=64 + ) + + def create_and_check_model(self, config, input_ids, attention_mask, pixel_values): + model = BLIPModel(config).to(torch_device).eval() + with torch.no_grad(): + result = model(input_ids, pixel_values, attention_mask) + self.parent.assertEqual( + result.logits_per_image.shape, (self.vision_model_tester.batch_size, self.text_model_tester.batch_size) + ) + self.parent.assertEqual( + result.logits_per_text.shape, (self.text_model_tester.batch_size, self.vision_model_tester.batch_size) + ) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + config, input_ids, attention_mask, pixel_values = config_and_inputs + inputs_dict = { + "input_ids": input_ids, + "attention_mask": attention_mask, + "pixel_values": pixel_values, + "return_loss": True, + } + return config, inputs_dict + + +@require_torch +class BLIPModelTest(ModelTesterMixin, unittest.TestCase): + all_model_classes = (BLIPModel,) if is_torch_available() else () + fx_compatible = False + test_head_masking = False + test_pruning = False + test_resize_embeddings = False + test_attention_outputs = False + + def setUp(self): + self.model_tester = BLIPModelTester(self) + + def test_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_model(*config_and_inputs) + + @unittest.skip(reason="Hidden_states is tested in individual model tests") + def test_hidden_states_output(self): + pass + + @unittest.skip(reason="Inputs_embeds is tested in individual model tests") + def test_inputs_embeds(self): + pass + + @unittest.skip(reason="Retain_grad is tested in individual model tests") + def test_retain_grad_hidden_states_attentions(self): + pass + + @unittest.skip(reason="BLIPModel does not have input/output embeddings") + def test_model_common_attributes(self): + pass + + # override as the `logit_scale` parameter initilization is different for BLIP + def test_initialization(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + configs_no_init = _config_zero_init(config) + for model_class in self.all_model_classes: + model = model_class(config=configs_no_init) + for name, param in model.named_parameters(): + if param.requires_grad: + # check if `logit_scale` is initilized as per the original implementation + if name == "logit_scale": + self.assertAlmostEqual( + param.data.item(), + np.log(1 / 0.07), + delta=1e-3, + msg=f"Parameter {name} of model {model_class} seems not properly initialized", + ) + else: + self.assertIn( + ((param.data.mean() * 1e9).round() / 1e9).item(), + [0.0, 1.0], + msg=f"Parameter {name} of model {model_class} seems not properly initialized", + ) + + def _create_and_check_torchscript(self, config, inputs_dict): + if not self.test_torchscript: + return + + configs_no_init = _config_zero_init(config) # To be sure we have no Nan + configs_no_init.torchscript = True + configs_no_init.return_dict = False + for model_class in self.all_model_classes: + model = model_class(config=configs_no_init) + model.to(torch_device) + model.eval() + + try: + input_ids = inputs_dict["input_ids"] + pixel_values = inputs_dict["pixel_values"] # BLIP needs pixel_values + traced_model = torch.jit.trace(model, (input_ids, pixel_values)) + except RuntimeError: + self.fail("Couldn't trace module.") + + with tempfile.TemporaryDirectory() as tmp_dir_name: + pt_file_name = os.path.join(tmp_dir_name, "traced_model.pt") + + try: + torch.jit.save(traced_model, pt_file_name) + except Exception: + self.fail("Couldn't save module.") + + try: + loaded_model = torch.jit.load(pt_file_name) + except Exception: + self.fail("Couldn't load module.") + + model.to(torch_device) + model.eval() + + loaded_model.to(torch_device) + loaded_model.eval() + + model_state_dict = model.state_dict() + loaded_model_state_dict = loaded_model.state_dict() + + self.assertEqual(set(model_state_dict.keys()), set(loaded_model_state_dict.keys())) + + models_equal = True + for layer_name, p1 in model_state_dict.items(): + p2 = loaded_model_state_dict[layer_name] + if p1.data.ne(p2.data).sum() > 0: + models_equal = False + + self.assertTrue(models_equal) + + def test_load_vision_text_config(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + # Save BLIPConfig and check if we can load BLIPVisionConfig from it + with tempfile.TemporaryDirectory() as tmp_dir_name: + config.save_pretrained(tmp_dir_name) + vision_config = BLIPVisionConfig.from_pretrained(tmp_dir_name) + self.assertDictEqual(config.vision_config.to_dict(), vision_config.to_dict()) + + # Save BLIPConfig and check if we can load BLIPTextConfig from it + with tempfile.TemporaryDirectory() as tmp_dir_name: + config.save_pretrained(tmp_dir_name) + text_config = BLIPTextConfig.from_pretrained(tmp_dir_name) + self.assertDictEqual(config.text_config.to_dict(), text_config.to_dict()) + + # overwrite from common since FlaxBLIPModel returns nested output + # which is not supported in the common test + @is_pt_flax_cross_test + def test_equivalence_pt_to_flax(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + with self.subTest(model_class.__name__): + + # load PyTorch class + pt_model = model_class(config).eval() + # Flax models don't use the `use_cache` option and cache is not returned as a default. + # So we disable `use_cache` here for PyTorch model. + pt_model.config.use_cache = False + + fx_model_class_name = "Flax" + model_class.__name__ + + if not hasattr(transformers, fx_model_class_name): + return + + fx_model_class = getattr(transformers, fx_model_class_name) + + # load Flax class + fx_model = fx_model_class(config, dtype=jnp.float32) + # make sure only flax inputs are forward that actually exist in function args + fx_input_keys = inspect.signature(fx_model.__call__).parameters.keys() + + # prepare inputs + pt_inputs = self._prepare_for_class(inputs_dict, model_class) + + # remove function args that don't exist in Flax + pt_inputs = {k: v for k, v in pt_inputs.items() if k in fx_input_keys} + + fx_state = convert_pytorch_state_dict_to_flax(pt_model.state_dict(), fx_model) + fx_model.params = fx_state + + with torch.no_grad(): + pt_outputs = pt_model(**pt_inputs).to_tuple() + + # convert inputs to Flax + fx_inputs = {k: np.array(v) for k, v in pt_inputs.items() if torch.is_tensor(v)} + fx_outputs = fx_model(**fx_inputs).to_tuple() + self.assertEqual(len(fx_outputs), len(pt_outputs), "Output lengths differ between Flax and PyTorch") + for fx_output, pt_output in zip(fx_outputs[:4], pt_outputs[:4]): + self.assert_almost_equals(fx_output, pt_output.numpy(), 4e-2) + + with tempfile.TemporaryDirectory() as tmpdirname: + pt_model.save_pretrained(tmpdirname) + fx_model_loaded = fx_model_class.from_pretrained(tmpdirname, from_pt=True) + + fx_outputs_loaded = fx_model_loaded(**fx_inputs).to_tuple() + self.assertEqual( + len(fx_outputs_loaded), len(pt_outputs), "Output lengths differ between Flax and PyTorch" + ) + for fx_output_loaded, pt_output in zip(fx_outputs_loaded[:4], pt_outputs[:4]): + self.assert_almost_equals(fx_output_loaded, pt_output.numpy(), 4e-2) + + # overwrite from common since FlaxBLIPModel returns nested output + # which is not supported in the common test + @is_pt_flax_cross_test + def test_equivalence_flax_to_pt(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + with self.subTest(model_class.__name__): + # load corresponding PyTorch class + pt_model = model_class(config).eval() + + # So we disable `use_cache` here for PyTorch model. + pt_model.config.use_cache = False + + fx_model_class_name = "Flax" + model_class.__name__ + + if not hasattr(transformers, fx_model_class_name): + # no flax model exists for this class + return + + fx_model_class = getattr(transformers, fx_model_class_name) + + # load Flax class + fx_model = fx_model_class(config, dtype=jnp.float32) + # make sure only flax inputs are forward that actually exist in function args + fx_input_keys = inspect.signature(fx_model.__call__).parameters.keys() + + pt_model = load_flax_weights_in_pytorch_model(pt_model, fx_model.params) + + # make sure weights are tied in PyTorch + pt_model.tie_weights() + + # prepare inputs + pt_inputs = self._prepare_for_class(inputs_dict, model_class) + + # remove function args that don't exist in Flax + pt_inputs = {k: v for k, v in pt_inputs.items() if k in fx_input_keys} + + with torch.no_grad(): + pt_outputs = pt_model(**pt_inputs).to_tuple() + + fx_inputs = {k: np.array(v) for k, v in pt_inputs.items() if torch.is_tensor(v)} + + fx_outputs = fx_model(**fx_inputs).to_tuple() + self.assertEqual(len(fx_outputs), len(pt_outputs), "Output lengths differ between Flax and PyTorch") + + for fx_output, pt_output in zip(fx_outputs[:4], pt_outputs[:4]): + self.assert_almost_equals(fx_output, pt_output.numpy(), 4e-2) + + with tempfile.TemporaryDirectory() as tmpdirname: + fx_model.save_pretrained(tmpdirname) + pt_model_loaded = model_class.from_pretrained(tmpdirname, from_flax=True) + + with torch.no_grad(): + pt_outputs_loaded = pt_model_loaded(**pt_inputs).to_tuple() + + self.assertEqual( + len(fx_outputs), len(pt_outputs_loaded), "Output lengths differ between Flax and PyTorch" + ) + for fx_output, pt_output in zip(fx_outputs[:4], pt_outputs_loaded[:4]): + self.assert_almost_equals(fx_output, pt_output.numpy(), 4e-2) + + @slow + def test_model_from_pretrained(self): + for model_name in BLIP_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: + model = BLIPModel.from_pretrained(model_name) + self.assertIsNotNone(model) + + +# We will verify our results on an image of cute cats +def prepare_img(): + url = "http://images.cocodataset.org/val2017/000000039769.jpg" + im = Image.open(requests.get(url, stream=True).raw) + return im + + +@require_vision +@require_torch +class BLIPModelIntegrationTest(unittest.TestCase): + @slow + def test_inference(self): + model_name = "ybelkada/blip-base" + model = BLIPModel.from_pretrained(model_name).to(torch_device) + processor = CLIPProcessor.from_pretrained(model_name) + + image = prepare_img() + inputs = processor( + text=["a photo of a cat", "a photo of a dog"], images=image, padding=True, return_tensors="pt" + ).to(torch_device) + + # forward pass + with torch.no_grad(): + outputs = model(**inputs) + + # verify the logits + self.assertEqual( + outputs.logits_per_image.shape, + torch.Size((inputs.pixel_values.shape[0], inputs.input_ids.shape[0])), + ) + self.assertEqual( + outputs.logits_per_text.shape, + torch.Size((inputs.input_ids.shape[0], inputs.pixel_values.shape[0])), + ) + + expected_logits = torch.tensor([[24.5701, 19.3049]], device=torch_device) + + self.assertTrue(torch.allclose(outputs.logits_per_image, expected_logits, atol=1e-3)) From afbc3bcd2de89e6e9d0b5e0add886c338744bf7c Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Fri, 9 Dec 2022 11:18:58 +0000 Subject: [PATCH 02/96] add v1 --- README.md | 1 + README_es.md | 1 + README_hd.md | 1 + README_ja.md | 1 + README_ko.md | 1 + README_zh-hans.md | 1 + README_zh-hant.md | 1 + docs/source/en/index.mdx | 2 + src/transformers/__init__.py | 57 +++--- src/transformers/models/__init__.py | 2 +- .../models/auto/configuration_auto.py | 6 +- .../models/auto/feature_extraction_auto.py | 2 +- .../models/auto/image_processing_auto.py | 2 +- src/transformers/models/auto/modeling_auto.py | 4 +- .../models/auto/processing_auto.py | 2 +- .../models/auto/tokenization_auto.py | 13 +- .../models/bert/configuration_bert.py | 2 + src/transformers/models/bert/modeling_bert.py | 12 +- src/transformers/models/blip/__init__.py | 39 ++-- .../models/blip/configuration_blip.py | 143 +++++---------- src/transformers/models/blip/modeling_blip.py | 173 ++++++++---------- 21 files changed, 196 insertions(+), 270 deletions(-) diff --git a/README.md b/README.md index 569466ec9598..adc84d175c8e 100644 --- a/README.md +++ b/README.md @@ -277,6 +277,7 @@ Current number of checkpoints: ![](https://img.shields.io/endpoint?url=https://h 1. **[BiT](https://huggingface.co/docs/transformers/main/model_doc/bit)** (from Google AI) released with the paper [Big Transfer (BiT): General Visual Representation Learning](https://arxiv.org/abs/1912.11370) by Alexander Kolesnikov, Lucas Beyer, Xiaohua Zhai, Joan Puigcerver, Jessica Yung, Sylvain Gelly, Neil Houlsby. 1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. 1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. +1. **[BLIP](https://huggingface.co/docs/transformers/main/model_doc/blip)** (from ) released with the paper []() by . 1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (from BigScience workshop) released by the [BigScience Workshop](https://bigscience.huggingface.co/). 1. **[BORT](https://huggingface.co/docs/transformers/model_doc/bort)** (from Alexa) released with the paper [Optimal Subarchitecture Extraction For BERT](https://arxiv.org/abs/2010.10499) by Adrian de Wynter and Daniel J. Perry. 1. **[ByT5](https://huggingface.co/docs/transformers/model_doc/byt5)** (from Google Research) released with the paper [ByT5: Towards a token-free future with pre-trained byte-to-byte models](https://arxiv.org/abs/2105.13626) by Linting Xue, Aditya Barua, Noah Constant, Rami Al-Rfou, Sharan Narang, Mihir Kale, Adam Roberts, Colin Raffel. diff --git a/README_es.md b/README_es.md index eabffc187c1d..85e17802fc6e 100644 --- a/README_es.md +++ b/README_es.md @@ -277,6 +277,7 @@ Número actual de puntos de control: ![](https://img.shields.io/endpoint?url=htt 1. **[BiT](https://huggingface.co/docs/transformers/main/model_doc/bit)** (from Google AI) released with the paper [Big Transfer (BiT) by Alexander Kolesnikov, Lucas Beyer, Xiaohua Zhai, Joan Puigcerver, Jessica Yung, Sylvain Gelly, Neil Houlsby. 1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. 1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. +1. **[BLIP](https://huggingface.co/docs/transformers/main/model_doc/blip)** (from ) released with the paper []() by . 1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (from BigScience workshop) released by the [BigScience Workshop](https://bigscience.huggingface.co/). 1. **[BORT](https://huggingface.co/docs/transformers/model_doc/bort)** (from Alexa) released with the paper [Optimal Subarchitecture Extraction For BERT](https://arxiv.org/abs/2010.10499) by Adrian de Wynter and Daniel J. Perry. 1. **[ByT5](https://huggingface.co/docs/transformers/model_doc/byt5)** (from Google Research) released with the paper [ByT5: Towards a token-free future with pre-trained byte-to-byte models](https://arxiv.org/abs/2105.13626) by Linting Xue, Aditya Barua, Noah Constant, Rami Al-Rfou, Sharan Narang, Mihir Kale, Adam Roberts, Colin Raffel. diff --git a/README_hd.md b/README_hd.md index 4fbfbbc91f42..c64e4c20a0bf 100644 --- a/README_hd.md +++ b/README_hd.md @@ -250,6 +250,7 @@ conda install -c huggingface transformers 1. **[BiT](https://huggingface.co/docs/transformers/main/model_doc/bit)** (from Google AI) released with the paper [Big Transfer (BiT) by Alexander Kolesnikov, Lucas Beyer, Xiaohua Zhai, Joan Puigcerver, Jessica Yung, Sylvain Gelly, Neil Houlsby. 1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (फेसबुक से) साथ में कागज [एक ओपन-डोमेन चैटबॉट बनाने की विधि](https://arxiv.org /abs/2004.13637) स्टीफन रोलर, एमिली दीनन, नमन गोयल, दा जू, मैरी विलियमसन, यिनहान लियू, जिंग जू, मायल ओट, कर्ट शस्टर, एरिक एम। स्मिथ, वाई-लैन बॉरो, जेसन वेस्टन द्वारा। 1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (फेसबुक से) साथ में पेपर [एक ओपन-डोमेन चैटबॉट बनाने की रेसिपी](https://arxiv .org/abs/2004.13637) स्टीफन रोलर, एमिली दीनन, नमन गोयल, दा जू, मैरी विलियमसन, यिनहान लियू, जिंग जू, मायल ओट, कर्ट शस्टर, एरिक एम स्मिथ, वाई-लैन बॉरो, जेसन वेस्टन द्वारा। +1. **[BLIP](https://huggingface.co/docs/transformers/main/model_doc/blip)** (from ) released with the paper []() by . 1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (from BigScience workshop) released by the [BigSicence Workshop](https://bigscience.huggingface.co/). 1. **[BORT](https://huggingface.co/docs/transformers/model_doc/bort)** (एलेक्सा से) कागज के साथ [बीईआरटी के लिए ऑप्टिमल सबआर्किटेक्चर एक्सट्रैक्शन](https://arxiv.org/abs/ 2010.10499) एड्रियन डी विंटर और डैनियल जे पेरी द्वारा। 1. **[ByT5](https://huggingface.co/docs/transformers/model_doc/byt5)** (Google अनुसंधान से) साथ में कागज [ByT5: पूर्व-प्रशिक्षित बाइट-टू-बाइट मॉडल के साथ एक टोकन-मुक्त भविष्य की ओर] (https://arxiv.org/abs/2105.13626) Linting Xue, Aditya Barua, Noah Constant, रामी अल-रफू, शरण नारंग, मिहिर काले, एडम रॉबर्ट्स, कॉलिन रैफेल द्वारा पोस्ट किया गया। diff --git a/README_ja.md b/README_ja.md index 34962668c18c..1bc6bbcd3d77 100644 --- a/README_ja.md +++ b/README_ja.md @@ -312,6 +312,7 @@ Flax、PyTorch、TensorFlowをcondaでインストールする方法は、それ 1. **[BiT](https://huggingface.co/docs/transformers/main/model_doc/bit)** (from Google AI) released with the paper [Big Transfer (BiT) by Alexander Kolesnikov, Lucas Beyer, Xiaohua Zhai, Joan Puigcerver, Jessica Yung, Sylvain Gelly, Neil Houlsby. 1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. 1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. +1. **[BLIP](https://huggingface.co/docs/transformers/main/model_doc/blip)** (from ) released with the paper []() by . 1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (from BigScience workshop) released by the [BigScience Workshop](https://bigscience.huggingface.co/). 1. **[BORT](https://huggingface.co/docs/transformers/model_doc/bort)** (from Alexa) released with the paper [Optimal Subarchitecture Extraction For BERT](https://arxiv.org/abs/2010.10499) by Adrian de Wynter and Daniel J. Perry. 1. **[ByT5](https://huggingface.co/docs/transformers/model_doc/byt5)** (from Google Research) released with the paper [ByT5: Towards a token-free future with pre-trained byte-to-byte models](https://arxiv.org/abs/2105.13626) by Linting Xue, Aditya Barua, Noah Constant, Rami Al-Rfou, Sharan Narang, Mihir Kale, Adam Roberts, Colin Raffel. diff --git a/README_ko.md b/README_ko.md index a17310b78885..5338db7e8b8c 100644 --- a/README_ko.md +++ b/README_ko.md @@ -227,6 +227,7 @@ Flax, PyTorch, TensorFlow 설치 페이지에서 이들을 conda로 설치하는 1. **[BiT](https://huggingface.co/docs/transformers/main/model_doc/bit)** (from Google AI) released with the paper [Big Transfer (BiT) by Alexander Kolesnikov, Lucas Beyer, Xiaohua Zhai, Joan Puigcerver, Jessica Yung, Sylvain Gelly, Neil Houlsby. 1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. 1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. +1. **[BLIP](https://huggingface.co/docs/transformers/main/model_doc/blip)** (from ) released with the paper []() by . 1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (from BigScience workshop) released by the [BigScience Workshop](https://bigscience.huggingface.co/). 1. **[BORT](https://huggingface.co/docs/transformers/model_doc/bort)** (from Alexa) released with the paper [Optimal Subarchitecture Extraction For BERT](https://arxiv.org/abs/2010.10499) by Adrian de Wynter and Daniel J. Perry. 1. **[ByT5](https://huggingface.co/docs/transformers/model_doc/byt5)** (from Google Research) released with the paper [ByT5: Towards a token-free future with pre-trained byte-to-byte models](https://arxiv.org/abs/2105.13626) by Linting Xue, Aditya Barua, Noah Constant, Rami Al-Rfou, Sharan Narang, Mihir Kale, Adam Roberts, Colin Raffel. diff --git a/README_zh-hans.md b/README_zh-hans.md index 53e1221f9077..0ba96c44e093 100644 --- a/README_zh-hans.md +++ b/README_zh-hans.md @@ -251,6 +251,7 @@ conda install -c huggingface transformers 1. **[BiT](https://huggingface.co/docs/transformers/main/model_doc/bit)** (来自 Google AI) 伴随论文 [Big Transfer (BiT) 由 Alexander Kolesnikov, Lucas Beyer, Xiaohua Zhai, Joan Puigcerver, Jessica Yung, Sylvain Gelly, Neil Houlsby 发布。 1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (来自 Facebook) 伴随论文 [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) 由 Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston 发布。 1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (来自 Facebook) 伴随论文 [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) 由 Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston 发布。 +1. **[BLIP](https://huggingface.co/docs/transformers/main/model_doc/blip)** (from ) released with the paper []() by . 1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (from BigScience workshop) released by the [BigScience Workshop](https://bigscience.huggingface.co/). 1. **[BORT](https://huggingface.co/docs/transformers/model_doc/bort)** (来自 Alexa) 伴随论文 [Optimal Subarchitecture Extraction For BERT](https://arxiv.org/abs/2010.10499) 由 Adrian de Wynter and Daniel J. Perry 发布。 1. **[ByT5](https://huggingface.co/docs/transformers/model_doc/byt5)** (来自 Google Research) 伴随论文 [ByT5: Towards a token-free future with pre-trained byte-to-byte models](https://arxiv.org/abs/2105.13626) 由 Linting Xue, Aditya Barua, Noah Constant, Rami Al-Rfou, Sharan Narang, Mihir Kale, Adam Roberts, Colin Raffel 发布。 diff --git a/README_zh-hant.md b/README_zh-hant.md index 6bf4cdaa48cb..63547dc647d9 100644 --- a/README_zh-hant.md +++ b/README_zh-hant.md @@ -263,6 +263,7 @@ conda install -c huggingface transformers 1. **[BiT](https://huggingface.co/docs/transformers/main/model_doc/bit)** (from Google AI) released with the paper [Big Transfer (BiT) by Alexander Kolesnikov, Lucas Beyer, Xiaohua Zhai, Joan Puigcerver, Jessica Yung, Sylvain Gelly, Neil Houlsby. 1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. 1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. +1. **[BLIP](https://huggingface.co/docs/transformers/main/model_doc/blip)** (from ) released with the paper []() by . 1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (from BigScience workshop) released by the [BigScience Workshop](https://bigscience.huggingface.co/). 1. **[BORT](https://huggingface.co/docs/transformers/model_doc/bort)** (from Alexa) released with the paper [Optimal Subarchitecture Extraction For BERT](https://arxiv.org/abs/2010.10499) by Adrian de Wynter and Daniel J. Perry. 1. **[ByT5](https://huggingface.co/docs/transformers/model_doc/byt5)** (from Google Research) released with the paper [ByT5: Towards a token-free future with pre-trained byte-to-byte models](https://arxiv.org/abs/2105.13626) by Linting Xue, Aditya Barua, Noah Constant, Rami Al-Rfou, Sharan Narang, Mihir Kale, Adam Roberts, Colin Raffel. diff --git a/docs/source/en/index.mdx b/docs/source/en/index.mdx index 544a74ac976a..4df907b5702f 100644 --- a/docs/source/en/index.mdx +++ b/docs/source/en/index.mdx @@ -64,6 +64,7 @@ The documentation is organized into five sections: 1. **[BiT](model_doc/bit)** (from Google AI) released with the paper [Big Transfer (BiT): General Visual Representation Learning](https://arxiv.org/abs/1912.11370) by Alexander Kolesnikov, Lucas Beyer, Xiaohua Zhai, Joan Puigcerver, Jessica Yung, Sylvain Gelly, Neil Houlsby. 1. **[Blenderbot](model_doc/blenderbot)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. 1. **[BlenderbotSmall](model_doc/blenderbot-small)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. +1. **[BLIP](model_doc/blip)** (from ) released with the paper []() by . 1. **[BLOOM](model_doc/bloom)** (from BigScience workshop) released by the [BigScience Workshop](https://bigscience.huggingface.co/). 1. **[BORT](model_doc/bort)** (from Alexa) released with the paper [Optimal Subarchitecture Extraction For BERT](https://arxiv.org/abs/2010.10499) by Adrian de Wynter and Daniel J. Perry. 1. **[ByT5](model_doc/byt5)** (from Google Research) released with the paper [ByT5: Towards a token-free future with pre-trained byte-to-byte models](https://arxiv.org/abs/2105.13626) by Linting Xue, Aditya Barua, Noah Constant, Rami Al-Rfou, Sharan Narang, Mihir Kale, Adam Roberts, Colin Raffel. @@ -236,6 +237,7 @@ Flax), PyTorch, and/or TensorFlow. | BiT | ❌ | ❌ | ✅ | ❌ | ❌ | | Blenderbot | ✅ | ✅ | ✅ | ✅ | ✅ | | BlenderbotSmall | ✅ | ✅ | ✅ | ✅ | ✅ | +| BLIP | ❌ | ❌ | ❌ | ❌ | ❌ | | BLOOM | ❌ | ✅ | ✅ | ❌ | ❌ | | CamemBERT | ✅ | ✅ | ✅ | ✅ | ❌ | | CANINE | ✅ | ❌ | ✅ | ❌ | ❌ | diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index faf996c997c3..ec956f27437f 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -168,6 +168,12 @@ "BlenderbotSmallConfig", "BlenderbotSmallTokenizer", ], + "models.blip": [ + "BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP", + "BlipConfig", + "BlipTextConfig", + "BlipVisionConfig", + ], "models.bloom": ["BLOOM_PRETRAINED_CONFIG_ARCHIVE_MAP", "BloomConfig"], "models.bort": [], "models.byt5": ["ByT5Tokenizer"], @@ -188,14 +194,6 @@ "CLIPTokenizer", "CLIPVisionConfig", ], - "models.blip": [ - "BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP", - "BLIPConfig", - - "BLIPTextConfig", - - "BLIPVisionConfig", - ], "models.clipseg": [ "CLIPSEG_PRETRAINED_CONFIG_ARCHIVE_MAP", "CLIPSegConfig", @@ -1094,6 +1092,14 @@ "BlenderbotSmallPreTrainedModel", ] ) + _import_structure["models.blip"].extend( + [ + "BLIP_PRETRAINED_MODEL_ARCHIVE_LIST", + "BlipForConditionalGeneration", + "BlipModel", + "BlipPreTrainedModel", + ] + ) _import_structure["models.bloom"].extend( [ "BLOOM_PRETRAINED_MODEL_ARCHIVE_LIST", @@ -1151,17 +1157,6 @@ "CLIPVisionModelWithProjection", ] ) - _import_structure["models.blip"].extend( - [ - "BLIP_PRETRAINED_MODEL_ARCHIVE_LIST", - "BLIPModel", - "BLIPPreTrainedModel", - "BLIPTextModel", - "BLIPTextModelWithProjection", - "BLIPVisionModel", - "BLIPVisionModelWithProjection", - ] - ) _import_structure["models.clipseg"].extend( [ "CLIPSEG_PRETRAINED_MODEL_ARCHIVE_LIST", @@ -3451,6 +3446,7 @@ BlenderbotSmallConfig, BlenderbotSmallTokenizer, ) + from .models.blip import BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP, BlipConfig, BlipTextConfig, BlipVisionConfig from .models.bloom import BLOOM_PRETRAINED_CONFIG_ARCHIVE_MAP, BloomConfig from .models.byt5 import ByT5Tokenizer from .models.camembert import CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, CamembertConfig @@ -3470,14 +3466,6 @@ CLIPTokenizer, CLIPVisionConfig, ) - from .models.blip import ( - BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP, - BLIPConfig, - - BLIPTextConfig, - - BLIPVisionConfig, - ) from .models.clipseg import ( CLIPSEG_PRETRAINED_CONFIG_ARCHIVE_MAP, CLIPSegConfig, @@ -4257,6 +4245,12 @@ BlenderbotSmallModel, BlenderbotSmallPreTrainedModel, ) + from .models.blip import ( + BLIP_PRETRAINED_MODEL_ARCHIVE_LIST, + BlipForConditionalGeneration, + BlipModel, + BlipPreTrainedModel, + ) from .models.bloom import ( BLOOM_PRETRAINED_MODEL_ARCHIVE_LIST, BloomForCausalLM, @@ -4304,15 +4298,6 @@ CLIPVisionModel, CLIPVisionModelWithProjection, ) - from .models.blip import ( - BLIP_PRETRAINED_MODEL_ARCHIVE_LIST, - BLIPModel, - BLIPPreTrainedModel, - BLIPTextModel, - BLIPTextModelWithProjection, - BLIPVisionModel, - BLIPVisionModelWithProjection, - ) from .models.clipseg import ( CLIPSEG_PRETRAINED_MODEL_ARCHIVE_LIST, CLIPSegForImageSegmentation, diff --git a/src/transformers/models/__init__.py b/src/transformers/models/__init__.py index fd645f3e294f..67de3e4ade6d 100644 --- a/src/transformers/models/__init__.py +++ b/src/transformers/models/__init__.py @@ -34,6 +34,7 @@ bit, blenderbot, blenderbot_small, + blip, bloom, bort, byt5, @@ -41,7 +42,6 @@ canine, chinese_clip, clip, - blip, clipseg, codegen, conditional_detr, diff --git a/src/transformers/models/auto/configuration_auto.py b/src/transformers/models/auto/configuration_auto.py index 8737454063c7..0b73ccc835f8 100644 --- a/src/transformers/models/auto/configuration_auto.py +++ b/src/transformers/models/auto/configuration_auto.py @@ -41,12 +41,12 @@ ("bit", "BitConfig"), ("blenderbot", "BlenderbotConfig"), ("blenderbot-small", "BlenderbotSmallConfig"), + ("blip", "BlipConfig"), ("bloom", "BloomConfig"), ("camembert", "CamembertConfig"), ("canine", "CanineConfig"), ("chinese_clip", "ChineseCLIPConfig"), ("clip", "CLIPConfig"), - ("blip", "BLIPConfig"), ("clipseg", "CLIPSegConfig"), ("codegen", "CodeGenConfig"), ("conditional_detr", "ConditionalDetrConfig"), @@ -197,12 +197,12 @@ ("bit", "BIT_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("blenderbot", "BLENDERBOT_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("blenderbot-small", "BLENDERBOT_SMALL_PRETRAINED_CONFIG_ARCHIVE_MAP"), + ("blip", "BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("bloom", "BLOOM_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("camembert", "CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("canine", "CANINE_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("chinese_clip", "CHINESE_CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("clip", "CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP"), - ("blip", "BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("clipseg", "CLIPSEG_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("codegen", "CODEGEN_PRETRAINED_CONFIG_ARCHIVE_MAP"), ("conditional_detr", "CONDITIONAL_DETR_PRETRAINED_CONFIG_ARCHIVE_MAP"), @@ -343,6 +343,7 @@ ("bit", "BiT"), ("blenderbot", "Blenderbot"), ("blenderbot-small", "BlenderbotSmall"), + ("blip", "BLIP"), ("bloom", "BLOOM"), ("bort", "BORT"), ("byt5", "ByT5"), @@ -350,7 +351,6 @@ ("canine", "CANINE"), ("chinese_clip", "Chinese-CLIP"), ("clip", "CLIP"), - ("blip", "BLIP"), ("clipseg", "CLIPSeg"), ("codegen", "CodeGen"), ("conditional_detr", "Conditional DETR"), diff --git a/src/transformers/models/auto/feature_extraction_auto.py b/src/transformers/models/auto/feature_extraction_auto.py index 8a30011b543c..4aeb0855edf8 100644 --- a/src/transformers/models/auto/feature_extraction_auto.py +++ b/src/transformers/models/auto/feature_extraction_auto.py @@ -39,9 +39,9 @@ [ ("audio-spectrogram-transformer", "ASTFeatureExtractor"), ("beit", "BeitFeatureExtractor"), + ("blip", "BLIPFeatureExtractor"), ("chinese_clip", "ChineseCLIPFeatureExtractor"), ("clip", "CLIPFeatureExtractor"), - ("blip", "BLIPFeatureExtractor"), ("clipseg", "ViTFeatureExtractor"), ("conditional_detr", "ConditionalDetrFeatureExtractor"), ("convnext", "ConvNextFeatureExtractor"), diff --git a/src/transformers/models/auto/image_processing_auto.py b/src/transformers/models/auto/image_processing_auto.py index 31aa17ec1be4..0fe4cb72972b 100644 --- a/src/transformers/models/auto/image_processing_auto.py +++ b/src/transformers/models/auto/image_processing_auto.py @@ -39,9 +39,9 @@ [ ("beit", "BeitImageProcessor"), ("bit", "BitImageProcessor"), + ("blip", "BLIPImageProcessor"), ("chinese_clip", "ChineseCLIPImageProcessor"), ("clip", "CLIPImageProcessor"), - ("blip", "BLIPImageProcessor"), ("clipseg", "ViTImageProcessor"), ("conditional_detr", "ConditionalDetrImageProcessor"), ("convnext", "ConvNextImageProcessor"), diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index a1b36020a2eb..91198c073e47 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -40,12 +40,12 @@ ("bit", "BitModel"), ("blenderbot", "BlenderbotModel"), ("blenderbot-small", "BlenderbotSmallModel"), + ("blip", "BLIPModel"), ("bloom", "BloomModel"), ("camembert", "CamembertModel"), ("canine", "CanineModel"), ("chinese_clip", "ChineseCLIPModel"), ("clip", "CLIPModel"), - ("blip", "BLIPModel"), ("clipseg", "CLIPSegModel"), ("codegen", "CodeGenModel"), ("conditional_detr", "ConditionalDetrModel"), @@ -850,9 +850,9 @@ _MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES = OrderedDict( [ # Model for Zero Shot Image Classification mapping + ("blip", "BLIPModel"), ("chinese_clip", "ChineseCLIPModel"), ("clip", "CLIPModel"), - ("blip", "BLIPModel"), ("clipseg", "CLIPSegModel"), ] ) diff --git a/src/transformers/models/auto/processing_auto.py b/src/transformers/models/auto/processing_auto.py index f72b481a2ebe..1a66b6d432b0 100644 --- a/src/transformers/models/auto/processing_auto.py +++ b/src/transformers/models/auto/processing_auto.py @@ -41,9 +41,9 @@ PROCESSOR_MAPPING_NAMES = OrderedDict( [ + ("blip", "BLIPProcessor"), ("chinese_clip", "ChineseCLIPProcessor"), ("clip", "CLIPProcessor"), - ("blip", "BLIPProcessor"), ("clipseg", "CLIPSegProcessor"), ("flava", "FlavaProcessor"), ("groupvit", "CLIPProcessor"), diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 43c3c2af3444..0489750c605e 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -77,6 +77,13 @@ ("biogpt", ("BioGptTokenizer", None)), ("blenderbot", ("BlenderbotTokenizer", "BlenderbotTokenizerFast")), ("blenderbot-small", ("BlenderbotSmallTokenizer", None)), + ( + "blip", + ( + "CLIPTokenizer", + "CLIPTokenizerFast" if is_tokenizers_available() else None, + ), + ), ("bloom", (None, "BloomTokenizerFast" if is_tokenizers_available() else None)), ("byt5", ("ByT5Tokenizer", None)), ( @@ -95,12 +102,6 @@ "CLIPTokenizerFast" if is_tokenizers_available() else None, ), ), - ( - "blip", - ( - "CLIPTokenizer", - "CLIPTokenizerFast" if is_tokenizers_available() else None, - ), ( "clipseg", ( diff --git a/src/transformers/models/bert/configuration_bert.py b/src/transformers/models/bert/configuration_bert.py index b2d64b7fde67..146175263ecd 100644 --- a/src/transformers/models/bert/configuration_bert.py +++ b/src/transformers/models/bert/configuration_bert.py @@ -156,6 +156,7 @@ def __init__( position_embedding_type="absolute", use_cache=True, classifier_dropout=None, + use_token_type_embed=True, **kwargs ): super().__init__(pad_token_id=pad_token_id, **kwargs) @@ -175,6 +176,7 @@ def __init__( self.position_embedding_type = position_embedding_type self.use_cache = use_cache self.classifier_dropout = classifier_dropout + self.use_token_type_embed = use_token_type_embed class BertOnnxConfig(OnnxConfig): diff --git a/src/transformers/models/bert/modeling_bert.py b/src/transformers/models/bert/modeling_bert.py index 16abb6c871ac..7e143466ffe9 100755 --- a/src/transformers/models/bert/modeling_bert.py +++ b/src/transformers/models/bert/modeling_bert.py @@ -185,7 +185,9 @@ def __init__(self, config): super().__init__() self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id) self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size) - self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.hidden_size) + self.token_type_embeddings = ( + nn.Embedding(config.type_vocab_size, config.hidden_size) if config.use_token_type_embed else None + ) # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load # any TensorFlow checkpoint file @@ -229,9 +231,13 @@ def forward( if inputs_embeds is None: inputs_embeds = self.word_embeddings(input_ids) - token_type_embeddings = self.token_type_embeddings(token_type_ids) - embeddings = inputs_embeds + token_type_embeddings + if self.token_type_embeddings is not None: + token_type_embeddings = self.token_type_embeddings(token_type_ids) + embeddings = inputs_embeds + token_type_embeddings + else: + embeddings = inputs_embeds + if self.position_embedding_type == "absolute": position_embeddings = self.position_embeddings(position_ids) embeddings += position_embeddings diff --git a/src/transformers/models/blip/__init__.py b/src/transformers/models/blip/__init__.py index 92f07d52b53e..95973da089fd 100644 --- a/src/transformers/models/blip/__init__.py +++ b/src/transformers/models/blip/__init__.py @@ -17,20 +17,15 @@ # limitations under the License. from typing import TYPE_CHECKING -from ...utils import ( - OptionalDependencyNotAvailable, - _LazyModule, - is_torch_available, -) +from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available _import_structure = { "configuration_blip": [ "BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP", - "BLIPConfig", - "BLIPOnnxConfig", - "BLIPTextConfig", - "BLIPVisionConfig", + "BlipConfig", + "BlipTextConfig", + "BlipVisionConfig", ], } @@ -42,22 +37,13 @@ else: _import_structure["modeling_blip"] = [ "BLIP_PRETRAINED_MODEL_ARCHIVE_LIST", - "BLIPModel", - "BLIPPreTrainedModel", - "BLIPTextModel", - "BLIPTextModelWithProjection", - "BLIPVisionModel", - "BLIPVisionModelWithProjection", + "BlipPModel", + "BlipPreTrainedModel", + "BlipForConditionalGeneration", ] if TYPE_CHECKING: - from .configuration_blip import ( - BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP, - BLIPConfig, - BLIPOnnxConfig, - BLIPTextConfig, - BLIPVisionConfig, - ) + from .configuration_blip import BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP, BlipConfig, BlipTextConfig, BlipVisionConfig try: if not is_torch_available(): @@ -67,12 +53,9 @@ else: from .modeling_blip import ( BLIP_PRETRAINED_MODEL_ARCHIVE_LIST, - BLIPModel, - BLIPPreTrainedModel, - BLIPTextModel, - BLIPTextModelWithProjection, - BLIPVisionModel, - BLIPVisionModelWithProjection, + BlipForConditionalGeneration, + BlipModel, + BlipPreTrainedModel, ) else: diff --git a/src/transformers/models/blip/configuration_blip.py b/src/transformers/models/blip/configuration_blip.py index 08da2af6eee2..4bf46bd52b80 100644 --- a/src/transformers/models/blip/configuration_blip.py +++ b/src/transformers/models/blip/configuration_blip.py @@ -16,16 +16,9 @@ import copy import os -from collections import OrderedDict -from typing import TYPE_CHECKING, Any, Mapping, Optional, Union - - -if TYPE_CHECKING: - from ...processing_utils import ProcessorMixin - from ...utils import TensorType +from typing import Union from ...configuration_utils import PretrainedConfig -from ...onnx import OnnxConfig from ...utils import logging @@ -36,8 +29,7 @@ } - -class BLIPTextConfig(PretrainedConfig): +class BlipTextConfig(PretrainedConfig): r""" This is the configuration class to store the configuration of a [`BLIPModel`]. It is used to instantiate an BLIP model according to the specified arguments, defining the model architecture. Instantiating a configuration with the @@ -80,10 +72,10 @@ class BLIPTextConfig(PretrainedConfig): Example: ```python - >>> from transformers import BLIPTextConfig, BLIPTextModel + >>> from transformers import BlipTextConfig, BLIPTextModel - >>> # Initializing a BLIPTextConfig with ybelkada/blip-base style configuration - >>> configuration = BLIPTextConfig() + >>> # Initializing a BlipTextConfig with ybelkada/blip-base style configuration + >>> configuration = BlipTextConfig() >>> # Initializing a BLIPTextModel (with random weights) from the ybelkada/blip-base style configuration >>> model = BLIPTextModel(configuration) @@ -95,31 +87,36 @@ class BLIPTextConfig(PretrainedConfig): def __init__( self, - vocab_size=49408, - hidden_size=512, - intermediate_size=2048, - projection_dim=512, + vocab_size=30524, + hidden_size=768, + intermediate_size=3072, + projection_dim=768, num_hidden_layers=12, num_attention_heads=8, - max_position_embeddings=77, + max_position_embeddings=512, hidden_act="quick_gelu", layer_norm_eps=0.00001, - dropout=0.0, - attention_dropout=0.0, + hidden_dropout_prob=0.0, + attention_probs_dropout_prob=0.0, initializer_range=0.02, initializer_factor=1.0, + type_vocab_size=3072, pad_token_id=1, bos_token_id=0, eos_token_id=2, + is_decoder=True, + add_cross_attention=True, + use_token_type_embed=False, **kwargs ): super().__init__(pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs) self.vocab_size = vocab_size + self.type_vocab_size = type_vocab_size self.hidden_size = hidden_size self.intermediate_size = intermediate_size self.projection_dim = projection_dim - self.dropout = dropout + self.hidden_dropout_prob = hidden_dropout_prob self.num_hidden_layers = num_hidden_layers self.num_attention_heads = num_attention_heads self.max_position_embeddings = max_position_embeddings @@ -127,14 +124,17 @@ def __init__( self.hidden_act = hidden_act self.initializer_range = initializer_range self.initializer_factor = initializer_factor - self.attention_dropout = attention_dropout + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.add_cross_attention = add_cross_attention + self.is_decoder = is_decoder + self.use_token_type_embed = use_token_type_embed @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) - # get the text config dict if we are loading from BLIPConfig + # get the text config dict if we are loading from BlipConfig if config_dict.get("model_type") == "blip": config_dict = config_dict["text_config"] @@ -147,7 +147,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], return cls.from_dict(config_dict, **kwargs) -class BLIPVisionConfig(PretrainedConfig): +class BlipVisionConfig(PretrainedConfig): r""" This is the configuration class to store the configuration of a [`BLIPModel`]. It is used to instantiate an BLIP model according to the specified arguments, defining the model architecture. Instantiating a configuration with the @@ -188,10 +188,10 @@ class BLIPVisionConfig(PretrainedConfig): Example: ```python - >>> from transformers import BLIPVisionConfig, BLIPVisionModel + >>> from transformers import BlipVisionConfig, BLIPVisionModel - >>> # Initializing a BLIPVisionConfig with ybelkada/blip-base style configuration - >>> configuration = BLIPVisionConfig() + >>> # Initializing a BlipVisionConfig with ybelkada/blip-base style configuration + >>> configuration = BlipVisionConfig() >>> # Initializing a BLIPVisionModel (with random weights) from the ybelkada/blip-base style configuration >>> model = BLIPVisionModel(configuration) @@ -210,8 +210,8 @@ def __init__( num_hidden_layers=12, num_attention_heads=12, num_channels=3, - image_size=224, - patch_size=32, + image_size=384, + patch_size=16, hidden_act="quick_gelu", layer_norm_eps=0.00001, dropout=0.0, @@ -242,7 +242,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) - # get the vision config dict if we are loading from BLIPConfig + # get the vision config dict if we are loading from BlipConfig if config_dict.get("model_type") == "blip": config_dict = config_dict["vision_config"] @@ -255,9 +255,9 @@ def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], return cls.from_dict(config_dict, **kwargs) -class BLIPConfig(PretrainedConfig): +class BlipConfig(PretrainedConfig): r""" - [`BLIPConfig`] is the configuration class to store the configuration of a [`BLIPModel`]. It is used to instantiate + [`BlipConfig`] is the configuration class to store the configuration of a [`BLIPModel`]. It is used to instantiate BLIP model according to the specified arguments, defining the text model and vision model configs. Instantiating a configuration with the defaults will yield a similar configuration to that of the BLIP [ybelkada/blip-base](https://huggingface.co/ybelkada/blip-base) architecture. @@ -267,9 +267,9 @@ class BLIPConfig(PretrainedConfig): Args: text_config (`dict`, *optional*): - Dictionary of configuration options used to initialize [`BLIPTextConfig`]. + Dictionary of configuration options used to initialize [`BlipTextConfig`]. vision_config (`dict`, *optional*): - Dictionary of configuration options used to initialize [`BLIPVisionConfig`]. + Dictionary of configuration options used to initialize [`BlipVisionConfig`]. projection_dim (`int`, *optional*, defaults to 512): Dimentionality of text and vision projection layers. logit_scale_init_value (`float`, *optional*, defaults to 2.6592): @@ -280,10 +280,10 @@ class BLIPConfig(PretrainedConfig): Example: ```python - >>> from transformers import BLIPConfig, BLIPModel + >>> from transformers import BlipConfig, BLIPModel - >>> # Initializing a BLIPConfig with ybelkada/blip-base style configuration - >>> configuration = BLIPConfig() + >>> # Initializing a BlipConfig with ybelkada/blip-base style configuration + >>> configuration = BlipConfig() >>> # Initializing a BLIPModel (with random weights) from the ybelkada/blip-base style configuration >>> model = BLIPModel(configuration) @@ -291,13 +291,13 @@ class BLIPConfig(PretrainedConfig): >>> # Accessing the model configuration >>> configuration = model.config - >>> # We can also initialize a BLIPConfig from a BLIPTextConfig and a BLIPVisionConfig + >>> # We can also initialize a BlipConfig from a BlipTextConfig and a BlipVisionConfig >>> # Initializing a BLIPText and BLIPVision configuration - >>> config_text = BLIPTextConfig() - >>> config_vision = BLIPVisionConfig() + >>> config_text = BlipTextConfig() + >>> config_vision = BlipVisionConfig() - >>> config = BLIPConfig.from_text_vision_configs(config_text, config_vision) + >>> config = BlipConfig.from_text_vision_configs(config_text, config_vision) ```""" model_type = "blip" @@ -318,27 +318,27 @@ def __init__( if text_config is None: text_config = {} - logger.info("text_config is None. Initializing the BLIPTextConfig with default values.") + logger.info("text_config is None. Initializing the BlipTextConfig with default values.") if vision_config is None: vision_config = {} - logger.info("vision_config is None. initializing the BLIPVisionConfig with default values.") + logger.info("vision_config is None. initializing the BlipVisionConfig with default values.") - self.text_config = BLIPTextConfig(**text_config) - self.vision_config = BLIPVisionConfig(**vision_config) + self.text_config = BlipTextConfig(**text_config) + self.vision_config = BlipVisionConfig(**vision_config) self.projection_dim = projection_dim self.logit_scale_init_value = logit_scale_init_value self.initializer_factor = 1.0 @classmethod - def from_text_vision_configs(cls, text_config: BLIPTextConfig, vision_config: BLIPVisionConfig, **kwargs): + def from_text_vision_configs(cls, text_config: BlipTextConfig, vision_config: BlipVisionConfig, **kwargs): r""" - Instantiate a [`BLIPConfig`] (or a derived class) from blip text model configuration and blip vision model + Instantiate a [`BlipConfig`] (or a derived class) from blip text model configuration and blip vision model configuration. Returns: - [`BLIPConfig`]: An instance of a configuration object + [`BlipConfig`]: An instance of a configuration object """ return cls(text_config=text_config.to_dict(), vision_config=vision_config.to_dict(), **kwargs) @@ -355,50 +355,3 @@ def to_dict(self): output["vision_config"] = self.vision_config.to_dict() output["model_type"] = self.__class__.model_type return output - - -class BLIPOnnxConfig(OnnxConfig): - @property - def inputs(self) -> Mapping[str, Mapping[int, str]]: - return OrderedDict( - [ - ("input_ids", {0: "batch", 1: "sequence"}), - ("pixel_values", {0: "batch", 1: "num_channels", 2: "height", 3: "width"}), - ("attention_mask", {0: "batch", 1: "sequence"}), - ] - ) - - @property - def outputs(self) -> Mapping[str, Mapping[int, str]]: - return OrderedDict( - [ - ("logits_per_image", {0: "batch"}), - ("logits_per_text", {0: "batch"}), - ("text_embeds", {0: "batch"}), - ("image_embeds", {0: "batch"}), - ] - ) - - @property - def atol_for_validation(self) -> float: - return 1e-4 - - def generate_dummy_inputs( - self, - processor: "ProcessorMixin", - batch_size: int = -1, - seq_length: int = -1, - framework: Optional["TensorType"] = None, - ) -> Mapping[str, Any]: - - text_input_dict = super().generate_dummy_inputs( - processor.tokenizer, batch_size=batch_size, seq_length=seq_length, framework=framework - ) - image_input_dict = super().generate_dummy_inputs( - processor.feature_extractor, batch_size=batch_size, framework=framework - ) - return {**text_input_dict, **image_input_dict} - - @property - def default_onnx_opset(self) -> int: - return 14 diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 4a27654190c9..535cbfa91fe2 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -22,6 +22,8 @@ import torch.utils.checkpoint from torch import nn +from transformers import BertLMHeadModel + from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import PreTrainedModel @@ -32,7 +34,7 @@ logging, replace_return_docstrings, ) -from .configuration_blip import BLIPConfig, BLIPTextConfig, BLIPVisionConfig +from .configuration_blip import BlipConfig, BlipTextConfig, BlipVisionConfig logger = logging.get_logger(__name__) @@ -45,7 +47,6 @@ ] - # Copied from transformers.models.bart.modeling_bart._expand_mask def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): """ @@ -76,7 +77,7 @@ def blip_loss(similarity: torch.Tensor) -> torch.Tensor: @dataclass # Copied from transformers.models.clip.modeling_clip.CLIPVisionModelOutput with CLIP->BLIP -class BLIPVisionModelOutput(ModelOutput): +class BlipVisionModelOutput(ModelOutput): """ Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states. @@ -105,8 +106,7 @@ class BLIPVisionModelOutput(ModelOutput): @dataclass -# Copied from transformers.models.clip.modeling_clip.CLIPTextModelOutput with CLIP->BLIP -class BLIPTextModelOutput(ModelOutput): +class BlipTextModelOutput(ModelOutput): """ Base class for text model's outputs that also contains a pooling of the last hidden states. @@ -135,8 +135,7 @@ class BLIPTextModelOutput(ModelOutput): @dataclass -# Copied from transformers.models.clip.modeling_clip.CLIPOutput with CLIP->BLIP -class BLIPOutput(ModelOutput): +class BlipOutput(ModelOutput): """ Args: loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `return_loss` is `True`): @@ -172,9 +171,8 @@ def to_tuple(self) -> Tuple[Any]: ) -# Copied from transformers.models.clip.modeling_clip.CLIPVisionEmbeddings with CLIP->BLIP -class BLIPVisionEmbeddings(nn.Module): - def __init__(self, config: BLIPVisionConfig): +class BlipVisionEmbeddings(nn.Module): + def __init__(self, config: BlipVisionConfig): super().__init__() self.config = config self.embed_dim = config.hidden_size @@ -203,9 +201,8 @@ def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: return embeddings -# Copied from transformers.models.clip.modeling_clip.CLIPTextEmbeddings with CLIP->BLIP -class BLIPTextEmbeddings(nn.Module): - def __init__(self, config: BLIPTextConfig): +class BlipTextEmbeddings(nn.Module): + def __init__(self, config: BlipTextConfig): super().__init__() embed_dim = config.hidden_size @@ -235,8 +232,7 @@ def forward( return embeddings -# Copied from transformers.models.clip.modeling_clip.CLIPAttention with CLIP->BLIP -class BLIPAttention(nn.Module): +class BlipAttention(nn.Module): """Multi-headed attention from 'Attention Is All You Need' paper""" def __init__(self, config): @@ -340,7 +336,6 @@ def forward( return attn_output, attn_weights_reshaped -# Copied from transformers.models.clip.modeling_clip.CLIPMLP with CLIP->BLIP class BLIPMLP(nn.Module): def __init__(self, config): super().__init__() @@ -356,12 +351,11 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: return hidden_states -# Copied from transformers.models.clip.modeling_clip.CLIPEncoderLayer with CLIP->BLIP -class BLIPEncoderLayer(nn.Module): - def __init__(self, config: BLIPConfig): +class BlipEncoderLayer(nn.Module): + def __init__(self, config: BlipConfig): super().__init__() self.embed_dim = config.hidden_size - self.self_attn = BLIPAttention(config) + self.self_attn = BlipAttention(config) self.layer_norm1 = nn.LayerNorm(self.embed_dim) self.mlp = BLIPMLP(config) self.layer_norm2 = nn.LayerNorm(self.embed_dim) @@ -407,14 +401,13 @@ def forward( return outputs -# Copied from transformers.models.clip.modeling_clip.CLIPPreTrainedModel with CLIP->BLIP,clip->blip -class BLIPPreTrainedModel(PreTrainedModel): +class BlipPreTrainedModel(PreTrainedModel): """ An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. """ - config_class = BLIPConfig + config_class = BlipConfig base_model_prefix = "blip" supports_gradient_checkpointing = True _keys_to_ignore_on_load_missing = [r"position_ids"] @@ -422,15 +415,15 @@ class BLIPPreTrainedModel(PreTrainedModel): def _init_weights(self, module): """Initialize the weights""" factor = self.config.initializer_factor - if isinstance(module, BLIPTextEmbeddings): + if isinstance(module, BlipTextEmbeddings): module.token_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02) module.position_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02) - elif isinstance(module, BLIPVisionEmbeddings): + elif isinstance(module, BlipVisionEmbeddings): factor = self.config.initializer_factor nn.init.normal_(module.class_embedding, mean=0.0, std=module.embed_dim**-0.5 * factor) nn.init.normal_(module.patch_embedding.weight, std=module.config.initializer_range * factor) nn.init.normal_(module.position_embedding.weight, std=module.config.initializer_range * factor) - elif isinstance(module, BLIPAttention): + elif isinstance(module, BlipAttention): factor = self.config.initializer_factor in_proj_std = (module.embed_dim**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor out_proj_std = (module.embed_dim**-0.5) * factor @@ -446,7 +439,7 @@ def _init_weights(self, module): fc_std = (2 * module.config.hidden_size) ** -0.5 * factor nn.init.normal_(module.fc1.weight, std=fc_std) nn.init.normal_(module.fc2.weight, std=in_proj_std) - elif isinstance(module, BLIPModel): + elif isinstance(module, BlipModel): nn.init.normal_( module.text_projection.weight, std=module.text_embed_dim**-0.5 * self.config.initializer_factor, @@ -455,11 +448,6 @@ def _init_weights(self, module): module.visual_projection.weight, std=module.vision_embed_dim**-0.5 * self.config.initializer_factor, ) - elif isinstance(module, BLIPVisionModelWithProjection): - nn.init.normal_( - module.visual_projection.weight, - std=self.config.hidden_size**-0.5 * self.config.initializer_factor, - ) elif isinstance(module, BLIPTextModelWithProjection): nn.init.normal_( module.text_projection.weight, @@ -473,7 +461,7 @@ def _init_weights(self, module): module.bias.data.zero_() def _set_gradient_checkpointing(self, module, value=False): - if isinstance(module, BLIPEncoder): + if isinstance(module, BlipEncoder): module.gradient_checkpointing = value @@ -487,7 +475,7 @@ def _set_gradient_checkpointing(self, module, value=False): and behavior. Parameters: - config ([`BLIPConfig`]): Model configuration class with all the parameters of the model. + config ([`BlipConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. """ @@ -578,19 +566,19 @@ def _set_gradient_checkpointing(self, module, value=False): # Copied from transformers.models.clip.modeling_clip.CLIPEncoder with CLIP->BLIP -class BLIPEncoder(nn.Module): +class BlipEncoder(nn.Module): """ Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a - [`BLIPEncoderLayer`]. + [`BlipEncoderLayer`]. Args: - config: BLIPConfig + config: BlipConfig """ - def __init__(self, config: BLIPConfig): + def __init__(self, config: BlipConfig): super().__init__() self.config = config - self.layers = nn.ModuleList([BLIPEncoderLayer(config) for _ in range(config.num_hidden_layers)]) + self.layers = nn.ModuleList([BlipEncoderLayer(config) for _ in range(config.num_hidden_layers)]) self.gradient_checkpointing = False def forward( @@ -682,16 +670,16 @@ def custom_forward(*inputs): class BLIPTextTransformer(nn.Module): - def __init__(self, config: BLIPTextConfig): + def __init__(self, config: BlipTextConfig): super().__init__() self.config = config embed_dim = config.hidden_size - self.embeddings = BLIPTextEmbeddings(config) - self.encoder = BLIPEncoder(config) + self.embeddings = BlipTextEmbeddings(config) + self.encoder = BlipEncoder(config) self.final_layer_norm = nn.LayerNorm(embed_dim) @add_start_docstrings_to_model_forward(BLIP_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=BLIPTextConfig) + @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=BlipTextConfig) def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -773,12 +761,12 @@ def _build_causal_attention_mask(self, bsz, seq_len, dtype): """The text model from BLIP without any head or projection on top.""", BLIP_START_DOCSTRING, ) -class BLIPTextModel(BLIPPreTrainedModel): - config_class = BLIPTextConfig +class BLIPTextModel(BlipPreTrainedModel): + config_class = BlipTextConfig - _no_split_modules = ["BLIPEncoderLayer"] + _no_split_modules = ["BlipEncoderLayer"] - def __init__(self, config: BLIPTextConfig): + def __init__(self, config: BlipTextConfig): super().__init__(config) self.text_model = BLIPTextTransformer(config) # Initialize weights and apply final processing @@ -791,7 +779,7 @@ def set_input_embeddings(self, value): self.text_model.embeddings.token_embedding = value @add_start_docstrings_to_model_forward(BLIP_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=BLIPTextConfig) + @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=BlipTextConfig) def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -830,19 +818,18 @@ def forward( ) -class BLIPVisionTransformer(nn.Module): - def __init__(self, config: BLIPVisionConfig): +class BlipVisionTransformer(nn.Module): + def __init__(self, config: BlipVisionConfig): super().__init__() self.config = config embed_dim = config.hidden_size - self.embeddings = BLIPVisionEmbeddings(config) - self.pre_layrnorm = nn.LayerNorm(embed_dim) - self.encoder = BLIPEncoder(config) + self.embeddings = BlipVisionEmbeddings(config) + self.encoder = BlipEncoder(config) self.post_layernorm = nn.LayerNorm(embed_dim) @add_start_docstrings_to_model_forward(BLIP_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=BLIPVisionConfig) + @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=BlipVisionConfig) def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -892,13 +879,13 @@ def forward( """The vision model from BLIP without any head or projection on top.""", BLIP_START_DOCSTRING, ) -class BLIPVisionModel(BLIPPreTrainedModel): - config_class = BLIPVisionConfig +class BLIPVisionModel(BlipPreTrainedModel): + config_class = BlipVisionConfig main_input_name = "pixel_values" - def __init__(self, config: BLIPVisionConfig): + def __init__(self, config: BlipVisionConfig): super().__init__(config) - self.vision_model = BLIPVisionTransformer(config) + self.vision_model = BlipVisionTransformer(config) # Initialize weights and apply final processing self.post_init() @@ -906,7 +893,7 @@ def get_input_embeddings(self) -> nn.Module: return self.vision_model.embeddings.patch_embedding @add_start_docstrings_to_model_forward(BLIP_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=BLIPVisionConfig) + @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=BlipVisionConfig) def forward( self, pixel_values: Optional[torch.FloatTensor] = None, @@ -947,21 +934,21 @@ def forward( @add_start_docstrings(BLIP_START_DOCSTRING) -class BLIPModel(BLIPPreTrainedModel): - config_class = BLIPConfig +class BlipModel(BlipPreTrainedModel): + config_class = BlipConfig - def __init__(self, config: BLIPConfig): + def __init__(self, config: BlipConfig): super().__init__(config) - if not isinstance(config.text_config, BLIPTextConfig): + if not isinstance(config.text_config, BlipTextConfig): raise ValueError( - "config.text_config is expected to be of type BLIPTextConfig but is of type" + "config.text_config is expected to be of type BlipTextConfig but is of type" f" {type(config.text_config)}." ) - if not isinstance(config.vision_config, BLIPVisionConfig): + if not isinstance(config.vision_config, BlipVisionConfig): raise ValueError( - "config.vision_config is expected to be of type BLIPVisionConfig but is of type" + "config.vision_config is expected to be of type BlipVisionConfig but is of type" f" {type(config.vision_config)}." ) @@ -973,7 +960,7 @@ def __init__(self, config: BLIPConfig): self.vision_embed_dim = vision_config.hidden_size self.text_model = BLIPTextTransformer(text_config) - self.vision_model = BLIPVisionTransformer(vision_config) + self.vision_model = BlipVisionTransformer(vision_config) self.visual_projection = nn.Linear(self.vision_embed_dim, self.projection_dim, bias=False) self.text_projection = nn.Linear(self.text_embed_dim, self.projection_dim, bias=False) @@ -1000,9 +987,9 @@ def get_text_features( Examples: ```python - >>> from transformers import CLIPTokenizer, BLIPModel + >>> from transformers import CLIPTokenizer, BlipModel - >>> model = BLIPModel.from_pretrained("ybelkada/blip-base") + >>> model = BlipModel.from_pretrained("ybelkada/blip-base") >>> tokenizer = CLIPTokenizer.from_pretrained("ybelkada/blip-base") >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") @@ -1047,9 +1034,9 @@ def get_image_features( ```python >>> from PIL import Image >>> import requests - >>> from transformers import CLIPProcessor, BLIPModel + >>> from transformers import CLIPProcessor, BlipModel - >>> model = BLIPModel.from_pretrained("ybelkada/blip-base") + >>> model = BlipModel.from_pretrained("ybelkada/blip-base") >>> processor = CLIPProcessor.from_pretrained("ybelkada/blip-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" @@ -1079,7 +1066,7 @@ def get_image_features( return image_features @add_start_docstrings_to_model_forward(BLIP_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BLIPOutput, config_class=BLIPConfig) + @replace_return_docstrings(output_type=BlipOutput, config_class=BlipConfig) def forward( self, input_ids: Optional[torch.LongTensor] = None, @@ -1090,7 +1077,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, BLIPOutput]: + ) -> Union[Tuple, BlipOutput]: r""" Returns: @@ -1099,9 +1086,9 @@ def forward( ```python >>> from PIL import Image >>> import requests - >>> from transformers import CLIPProcessor, BLIPModel + >>> from transformers import CLIPProcessor, BlipModel - >>> model = BLIPModel.from_pretrained("ybelkada/blip-base") + >>> model = BlipModel.from_pretrained("ybelkada/blip-base") >>> processor = CLIPProcessor.from_pretrained("ybelkada/blip-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" @@ -1161,7 +1148,7 @@ def forward( output = (logits_per_image, logits_per_text, text_embeds, image_embeds, text_outputs, vision_outputs) return ((loss,) + output) if loss is not None else output - return BLIPOutput( + return BlipOutput( loss=loss, logits_per_image=logits_per_image, logits_per_text=logits_per_text, @@ -1178,12 +1165,12 @@ def forward( """, BLIP_START_DOCSTRING, ) -class BLIPTextModelWithProjection(BLIPPreTrainedModel): - config_class = BLIPTextConfig +class BLIPTextModelWithProjection(BlipPreTrainedModel): + config_class = BlipTextConfig - _no_split_modules = ["BLIPEncoderLayer"] + _no_split_modules = ["BlipEncoderLayer"] - def __init__(self, config: BLIPTextConfig): + def __init__(self, config: BlipTextConfig): super().__init__(config) self.text_model = BLIPTextTransformer(config) @@ -1200,7 +1187,7 @@ def set_input_embeddings(self, value): self.text_model.embeddings.token_embedding = value @add_start_docstrings_to_model_forward(BLIP_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BLIPTextModelOutput, config_class=BLIPTextConfig) + @replace_return_docstrings(output_type=BlipTextModelOutput, config_class=BlipTextConfig) def forward( self, input_ids: Optional[torch.Tensor] = None, @@ -1209,7 +1196,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, BLIPTextModelOutput]: + ) -> Union[Tuple, BlipTextModelOutput]: r""" Returns: @@ -1245,7 +1232,7 @@ def forward( outputs = (text_embeds, text_outputs[0]) + text_outputs[2:] return tuple(output for output in outputs if output is not None) - return BLIPTextModelOutput( + return BlipTextModelOutput( text_embeds=text_embeds, last_hidden_state=text_outputs.last_hidden_state, hidden_states=text_outputs.hidden_states, @@ -1259,16 +1246,16 @@ def forward( """, BLIP_START_DOCSTRING, ) -class BLIPVisionModelWithProjection(BLIPPreTrainedModel): - config_class = BLIPVisionConfig +class BlipForConditionalGeneration(BlipPreTrainedModel): + config_class = BlipVisionConfig main_input_name = "pixel_values" - def __init__(self, config: BLIPVisionConfig): + def __init__(self, config: BlipConfig): super().__init__(config) - self.vision_model = BLIPVisionTransformer(config) + self.vision_model = BlipVisionTransformer(config.vision_config) - self.visual_projection = nn.Linear(config.hidden_size, config.projection_dim, bias=False) + self.text_decoder = BertLMHeadModel(config.text_config) # Initialize weights and apply final processing self.post_init() @@ -1277,14 +1264,14 @@ def get_input_embeddings(self) -> nn.Module: return self.vision_model.embeddings.patch_embedding @add_start_docstrings_to_model_forward(BLIP_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BLIPVisionModelOutput, config_class=BLIPVisionConfig) + @replace_return_docstrings(output_type=BlipVisionModelOutput, config_class=BlipVisionConfig) def forward( self, pixel_values: Optional[torch.FloatTensor] = None, output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, BLIPVisionModelOutput]: + ) -> Union[Tuple, BlipVisionModelOutput]: r""" Returns: @@ -1293,9 +1280,9 @@ def forward( ```python >>> from PIL import Image >>> import requests - >>> from transformers import CLIPProcessor, BLIPVisionModelWithProjection + >>> from transformers import CLIPProcessor, BLIPForImageCaptioning - >>> model = BLIPVisionModelWithProjection.from_pretrained("ybelkada/blip-base") + >>> model = BLIPForImageCaptioning.from_pretrained("ybelkada/blip-base") >>> processor = CLIPProcessor.from_pretrained("ybelkada/blip-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" @@ -1323,7 +1310,7 @@ def forward( outputs = (image_embeds, vision_outputs[0]) + vision_outputs[2:] return tuple(output for output in outputs if output is not None) - return BLIPVisionModelOutput( + return BlipVisionModelOutput( image_embeds=image_embeds, last_hidden_state=vision_outputs.last_hidden_state, hidden_states=vision_outputs.hidden_states, From d21be076cee52dbd173b4b9603191e707a86646b Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Fri, 9 Dec 2022 11:19:42 +0000 Subject: [PATCH 03/96] v1 --- src/transformers/models/blip/modeling_blip.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 535cbfa91fe2..c6c9d7e798ef 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -565,7 +565,6 @@ def _set_gradient_checkpointing(self, module, value=False): """ -# Copied from transformers.models.clip.modeling_clip.CLIPEncoder with CLIP->BLIP class BlipEncoder(nn.Module): """ Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a From 365fe148a882d1644a42840983d3b5197974bacd Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Fri, 9 Dec 2022 11:27:35 +0000 Subject: [PATCH 04/96] v1 --- src/transformers/models/blip/modeling_blip.py | 26 +++++++------------ 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index c6c9d7e798ef..209e744a70ae 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -179,16 +179,15 @@ def __init__(self, config: BlipVisionConfig): self.image_size = config.image_size self.patch_size = config.patch_size - self.class_embedding = nn.Parameter(torch.randn(self.embed_dim)) + self.class_embedding = nn.Parameter(torch.randn(1, 1, self.embed_dim)) self.patch_embedding = nn.Conv2d( - in_channels=3, out_channels=self.embed_dim, kernel_size=self.patch_size, stride=self.patch_size, bias=False + in_channels=3, out_channels=self.embed_dim, kernel_size=self.patch_size, stride=self.patch_size ) self.num_patches = (self.image_size // self.patch_size) ** 2 self.num_positions = self.num_patches + 1 - self.position_embedding = nn.Embedding(self.num_positions, self.embed_dim) - self.register_buffer("position_ids", torch.arange(self.num_positions).expand((1, -1))) + self.position_embedding = nn.Parameter(torch.zeros(1, self.num_positions, self.embed_dim)) def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: batch_size = pixel_values.shape[0] @@ -249,10 +248,9 @@ def __init__(self, config): self.scale = self.head_dim**-0.5 self.dropout = config.attention_dropout - self.k_proj = nn.Linear(self.embed_dim, self.embed_dim) - self.v_proj = nn.Linear(self.embed_dim, self.embed_dim) - self.q_proj = nn.Linear(self.embed_dim, self.embed_dim) - self.out_proj = nn.Linear(self.embed_dim, self.embed_dim) + self.qkv = nn.Linear(self.embed_dim, 3 * self.embed_dim) + + self.proj = nn.Linear(self.embed_dim, self.embed_dim) def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() @@ -336,7 +334,7 @@ def forward( return attn_output, attn_weights_reshaped -class BLIPMLP(nn.Module): +class BlipMLP(nn.Module): def __init__(self, config): super().__init__() self.config = config @@ -357,7 +355,7 @@ def __init__(self, config: BlipConfig): self.embed_dim = config.hidden_size self.self_attn = BlipAttention(config) self.layer_norm1 = nn.LayerNorm(self.embed_dim) - self.mlp = BLIPMLP(config) + self.mlp = BlipMLP(config) self.layer_norm2 = nn.LayerNorm(self.embed_dim) def forward( @@ -422,16 +420,12 @@ def _init_weights(self, module): factor = self.config.initializer_factor nn.init.normal_(module.class_embedding, mean=0.0, std=module.embed_dim**-0.5 * factor) nn.init.normal_(module.patch_embedding.weight, std=module.config.initializer_range * factor) - nn.init.normal_(module.position_embedding.weight, std=module.config.initializer_range * factor) elif isinstance(module, BlipAttention): factor = self.config.initializer_factor in_proj_std = (module.embed_dim**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor out_proj_std = (module.embed_dim**-0.5) * factor - nn.init.normal_(module.q_proj.weight, std=in_proj_std) - nn.init.normal_(module.k_proj.weight, std=in_proj_std) - nn.init.normal_(module.v_proj.weight, std=in_proj_std) - nn.init.normal_(module.out_proj.weight, std=out_proj_std) - elif isinstance(module, BLIPMLP): + nn.init.normal_(module.qkv.weight, std=in_proj_std) + elif isinstance(module, BlipMLP): factor = self.config.initializer_factor in_proj_std = ( (module.config.hidden_size**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor From 4538c2447ce1ad4f177792a50a9c3aa7d1d08d3e Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Fri, 9 Dec 2022 12:44:00 +0000 Subject: [PATCH 05/96] vision encoder logits match --- .../models/blip/configuration_blip.py | 12 +- src/transformers/models/blip/modeling_blip.py | 129 ++++++++---------- 2 files changed, 64 insertions(+), 77 deletions(-) diff --git a/src/transformers/models/blip/configuration_blip.py b/src/transformers/models/blip/configuration_blip.py index 4bf46bd52b80..00e78414726d 100644 --- a/src/transformers/models/blip/configuration_blip.py +++ b/src/transformers/models/blip/configuration_blip.py @@ -55,9 +55,9 @@ class BlipTextConfig(PretrainedConfig): max_position_embeddings (`int`, *optional*, defaults to 77): The maximum sequence length that this model might ever be used with. Typically set this to something large just in case (e.g., 512 or 1024 or 2048). - hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`): + hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`): The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, - `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported. layer_norm_eps (`float`, *optional*, + `"relu"`, `"selu"` and `"gelu_new"` ``"gelu"` are supported. layer_norm_eps (`float`, *optional*, defaults to 1e-5): The epsilon used by the layer normalization layers. attention_dropout (`float`, *optional*, defaults to 0.0): The dropout ratio for the attention probabilities. @@ -94,7 +94,7 @@ def __init__( num_hidden_layers=12, num_attention_heads=8, max_position_embeddings=512, - hidden_act="quick_gelu", + hidden_act="gelu", layer_norm_eps=0.00001, hidden_dropout_prob=0.0, attention_probs_dropout_prob=0.0, @@ -171,9 +171,9 @@ class BlipVisionConfig(PretrainedConfig): The size (resolution) of each image. patch_size (`int`, *optional*, defaults to 32): The size (resolution) of each patch. - hidden_act (`str` or `function`, *optional*, defaults to `"quick_gelu"`): + hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`): The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, - `"relu"`, `"selu"` and `"gelu_new"` ``"quick_gelu"` are supported. layer_norm_eps (`float`, *optional*, + `"relu"`, `"selu"` and `"gelu_new"` ``"gelu"` are supported. layer_norm_eps (`float`, *optional*, defaults to 1e-5): The epsilon used by the layer normalization layers. dropout (`float`, *optional*, defaults to 0.0): The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler. @@ -212,7 +212,7 @@ def __init__( num_channels=3, image_size=384, patch_size=16, - hidden_act="quick_gelu", + hidden_act="gelu", layer_norm_eps=0.00001, dropout=0.0, attention_dropout=0.0, diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 209e744a70ae..0c2767fd0841 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """ PyTorch BLIP model.""" - +import math from dataclasses import dataclass from typing import Any, Optional, Tuple, Union @@ -196,7 +196,7 @@ def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: class_embeds = self.class_embedding.expand(batch_size, 1, -1) embeddings = torch.cat([class_embeds, patch_embeds], dim=1) - embeddings = embeddings + self.position_embedding(self.position_ids) + embeddings = embeddings + self.position_embedding[:, :embeddings.size(1), :] return embeddings @@ -246,7 +246,7 @@ def __init__(self, config): f" {self.num_heads})." ) self.scale = self.head_dim**-0.5 - self.dropout = config.attention_dropout + self.dropout = nn.Dropout(config.attention_dropout) self.qkv = nn.Linear(self.embed_dim, 3 * self.embed_dim) @@ -254,84 +254,64 @@ def __init__(self, config): def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def _split_heads(self, fused_qkv: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """ + Split the last dimension into (num_heads, head_dim) without making any copies, results share same memory + storage as `fused_qkv` + + Args: + fused_qkv (`torch.tensor`, *required*): [batch_size, seq_length, num_heads * 3 * head_dim] + + Returns: + query: [batch_size, seq_length, num_heads, head_dim] key: [batch_size, seq_length, num_heads, head_dim] + value: [batch_size, seq_length, num_heads, head_dim] + """ + batch_size, seq_length, three_times_hidden_size = fused_qkv.shape + fused_qkv = fused_qkv.view(batch_size, seq_length, self.num_heads, 3, self.head_dim) + return fused_qkv[..., 0, :], fused_qkv[..., 1, :], fused_qkv[..., 2, :] def forward( self, hidden_states: torch.Tensor, - attention_mask: Optional[torch.Tensor] = None, - causal_attention_mask: Optional[torch.Tensor] = None, + head_mask: Optional[torch.Tensor] = None, output_attentions: Optional[bool] = False, ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: """Input shape: Batch x Time x Channel""" bsz, tgt_len, embed_dim = hidden_states.size() - # get query proj - query_states = self.q_proj(hidden_states) * self.scale - key_states = self._shape(self.k_proj(hidden_states), -1, bsz) - value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + mixed_qkv = self.qkv(hidden_states) + # query_states, key_states, value_states = self._split_heads(mixed_qkv) + qkv = self.qkv(hidden_states).reshape(bsz, tgt_len, 3, self.num_heads, embed_dim // self.num_heads).permute(2, 0, 3, 1, 4) + query_states, key_states, value_states = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple) - proj_shape = (bsz * self.num_heads, -1, self.head_dim) - query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) - key_states = key_states.view(*proj_shape) - value_states = value_states.view(*proj_shape) + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_states, key_states.transpose(-1, -2)) - src_len = key_states.size(1) - attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + attention_scores = attention_scores * self.scale - if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): - raise ValueError( - f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is" - f" {attn_weights.size()}" - ) + # Normalize the attention scores to probabilities. + attention_probs = nn.functional.softmax(attention_scores, dim=-1) - # apply the causal_attention_mask first - if causal_attention_mask is not None: - if causal_attention_mask.size() != (bsz, 1, tgt_len, src_len): - raise ValueError( - f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is" - f" {causal_attention_mask.size()}" - ) - attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + causal_attention_mask - attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) - if attention_mask is not None: - if attention_mask.size() != (bsz, 1, tgt_len, src_len): - raise ValueError( - f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" - ) - attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask - attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask - attn_weights = nn.functional.softmax(attn_weights, dim=-1) + context_layer = torch.matmul(attention_probs, value_states).permute(0, 2, 1, 3) - if output_attentions: - # this operation is a bit akward, but it's required to - # make sure that attn_weights keeps its gradient. - # In order to do so, attn_weights have to reshaped - # twice and have to be reused in the following - attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) - attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) - else: - attn_weights_reshaped = None + new_context_layer_shape = context_layer.size()[:-2] + (self.embed_dim,) + context_layer = context_layer.reshape(new_context_layer_shape) - attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + output = self.proj(context_layer) - attn_output = torch.bmm(attn_probs, value_states) + outputs = (output, attention_probs) if output_attentions else (output, None) - if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): - raise ValueError( - f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is" - f" {attn_output.size()}" - ) - - attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim) - attn_output = attn_output.transpose(1, 2) - attn_output = attn_output.reshape(bsz, tgt_len, embed_dim) - - attn_output = self.out_proj(attn_output) - - return attn_output, attn_weights_reshaped + return outputs class BlipMLP(nn.Module): @@ -362,7 +342,6 @@ def forward( self, hidden_states: torch.Tensor, attention_mask: torch.Tensor, - causal_attention_mask: torch.Tensor, output_attentions: Optional[bool] = False, ) -> Tuple[torch.FloatTensor]: """ @@ -380,16 +359,15 @@ def forward( hidden_states = self.layer_norm1(hidden_states) hidden_states, attn_weights = self.self_attn( hidden_states=hidden_states, - attention_mask=attention_mask, - causal_attention_mask=causal_attention_mask, + head_mask=attention_mask, output_attentions=output_attentions, ) - hidden_states = residual + hidden_states - + hidden_states = hidden_states + residual residual = hidden_states hidden_states = self.layer_norm2(hidden_states) hidden_states = self.mlp(hidden_states) - hidden_states = residual + hidden_states + + hidden_states = hidden_states + residual outputs = (hidden_states,) @@ -643,7 +621,6 @@ def custom_forward(*inputs): layer_outputs = encoder_layer( hidden_states, attention_mask, - causal_attention_mask, output_attentions=output_attentions, ) @@ -844,7 +821,6 @@ def forward( raise ValueError("You have to specify pixel_values") hidden_states = self.embeddings(pixel_values) - hidden_states = self.pre_layrnorm(hidden_states) encoder_outputs = self.encoder( inputs_embeds=hidden_states, @@ -854,6 +830,8 @@ def forward( ) last_hidden_state = encoder_outputs[0] + last_hidden_state = self.post_layernorm(last_hidden_state) + pooled_output = last_hidden_state[:, 0, :] pooled_output = self.post_layernorm(pooled_output) @@ -1261,6 +1239,7 @@ def get_input_embeddings(self) -> nn.Module: def forward( self, pixel_values: Optional[torch.FloatTensor] = None, + input_ids: Optional[torch.LongTensor] = None, output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, @@ -1295,9 +1274,17 @@ def forward( return_dict=return_dict, ) - pooled_output = vision_outputs[1] # pooled_output + image_embeds = vision_outputs[1] + + if input_ids is not None: + # Case 1: image captioning + decoder_targets = input_ids.masked_fill(input_ids == self.tokenizer.pad_token_id, -100) - image_embeds = self.visual_projection(pooled_output) + outputs = self.text_decoder( + input_ids, + encoder_hidden_states=image_embeds, + labels=decoder_targets, + ) if not return_dict: outputs = (image_embeds, vision_outputs[0]) + vision_outputs[2:] From d6ff6e02560f41e1d1d3fc77f087d85bbd1ab40e Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Fri, 9 Dec 2022 15:54:15 +0000 Subject: [PATCH 06/96] v2 --- .../models/blip/configuration_blip.py | 11 +- src/transformers/models/blip/modeling_blip.py | 53 +- .../models/blip/modeling_blip_text.py | 944 ++++++++++++++++++ 3 files changed, 997 insertions(+), 11 deletions(-) create mode 100644 src/transformers/models/blip/modeling_blip_text.py diff --git a/src/transformers/models/blip/configuration_blip.py b/src/transformers/models/blip/configuration_blip.py index 00e78414726d..7bc5ed83e6a9 100644 --- a/src/transformers/models/blip/configuration_blip.py +++ b/src/transformers/models/blip/configuration_blip.py @@ -95,21 +95,23 @@ def __init__( num_attention_heads=8, max_position_embeddings=512, hidden_act="gelu", - layer_norm_eps=0.00001, + layer_norm_eps=1e-12, hidden_dropout_prob=0.0, attention_probs_dropout_prob=0.0, initializer_range=0.02, initializer_factor=1.0, type_vocab_size=3072, - pad_token_id=1, - bos_token_id=0, + pad_token_id=0, + bos_token_id=30522, eos_token_id=2, is_decoder=True, add_cross_attention=True, use_token_type_embed=False, + sep_token_id=102, + use_cache=True, **kwargs ): - super().__init__(pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs) + super().__init__(pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, sep_token_id=sep_token_id, **kwargs) self.vocab_size = vocab_size self.type_vocab_size = type_vocab_size @@ -128,6 +130,7 @@ def __init__( self.add_cross_attention = add_cross_attention self.is_decoder = is_decoder self.use_token_type_embed = use_token_type_embed + self.use_cache = use_cache @classmethod def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig": diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 0c2767fd0841..3b429c82afc3 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -22,7 +22,6 @@ import torch.utils.checkpoint from torch import nn -from transformers import BertLMHeadModel from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling @@ -35,6 +34,7 @@ replace_return_docstrings, ) from .configuration_blip import BlipConfig, BlipTextConfig, BlipVisionConfig +from .modeling_blip_text import BlipTextLMHeadModel logger = logging.get_logger(__name__) @@ -831,7 +831,7 @@ def forward( last_hidden_state = encoder_outputs[0] last_hidden_state = self.post_layernorm(last_hidden_state) - + pooled_output = last_hidden_state[:, 0, :] pooled_output = self.post_layernorm(pooled_output) @@ -1226,7 +1226,9 @@ def __init__(self, config: BlipConfig): self.vision_model = BlipVisionTransformer(config.vision_config) - self.text_decoder = BertLMHeadModel(config.text_config) + self.text_decoder = BlipTextLMHeadModel(config.text_config) + + self.decoder_input_ids = config.text_config.bos_token_id # Initialize weights and apply final processing self.post_init() @@ -1238,10 +1240,11 @@ def get_input_embeddings(self) -> nn.Module: @replace_return_docstrings(output_type=BlipVisionModelOutput, config_class=BlipVisionConfig) def forward( self, - pixel_values: Optional[torch.FloatTensor] = None, - input_ids: Optional[torch.LongTensor] = None, + input_ids: torch.LongTensor, + pixel_values: torch.FloatTensor, output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, + decoder_input_ids: Optional[int] = 0, return_dict: Optional[bool] = None, ) -> Union[Tuple, BlipVisionModelOutput]: r""" @@ -1274,11 +1277,11 @@ def forward( return_dict=return_dict, ) - image_embeds = vision_outputs[1] + image_embeds = vision_outputs[0] if input_ids is not None: # Case 1: image captioning - decoder_targets = input_ids.masked_fill(input_ids == self.tokenizer.pad_token_id, -100) + decoder_targets = input_ids.masked_fill(input_ids == self.decoder_input_ids, -100) outputs = self.text_decoder( input_ids, @@ -1296,3 +1299,39 @@ def forward( hidden_states=vision_outputs.hidden_states, attentions=vision_outputs.attentions, ) + + def generate(self, + input_ids: torch.LongTensor, + pixel_values: torch.FloatTensor, + **generate_kwargs + ): + r""" + Overrides `generate` function to be able to use the model as a conditional generator + + Args: + `input_ids` + """ + vision_outputs = self.vision_model( + pixel_values=pixel_values, + ) + + image_embeds = vision_outputs[0] + + image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image_embeds.device) + model_kwargs = {"encoder_hidden_states": image_embeds, "encoder_attention_mask":image_atts} + + if isinstance(input_ids, list): + input_ids = torch.LongTensor(input_ids) + + input_ids[:, 0] = self.config.text_config.bos_token_id + + outputs = self.text_decoder.generate( + input_ids=input_ids[:, :-1], + eos_token_id=self.config.text_config.sep_token_id, + pad_token_id=self.config.text_config.pad_token_id, + **generate_kwargs, + **model_kwargs, + ) + + return outputs + diff --git a/src/transformers/models/blip/modeling_blip_text.py b/src/transformers/models/blip/modeling_blip_text.py new file mode 100644 index 000000000000..8145d25ace6f --- /dev/null +++ b/src/transformers/models/blip/modeling_blip_text.py @@ -0,0 +1,944 @@ +''' + * Copyright (c) 2022, salesforce.com, inc. + * All rights reserved. + * SPDX-License-Identifier: BSD-3-Clause + * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause + * By Junnan Li + * Based on huggingface code base + * https://github.com/huggingface/transformers/blob/v4.15.0/src/transformers/models/bert +''' + +import math +from typing import Tuple + +import torch +from torch import Tensor, device, nn +import torch.utils.checkpoint +from torch import nn +from torch.nn import CrossEntropyLoss + +from transformers.activations import ACT2FN + +from transformers.modeling_outputs import ( + BaseModelOutputWithPastAndCrossAttentions, + BaseModelOutputWithPoolingAndCrossAttentions, + CausalLMOutputWithCrossAttentions, +) +from transformers.modeling_utils import ( + PreTrainedModel, + apply_chunking_to_forward, + find_pruneable_heads_and_indices, + prune_linear_layer, +) +from transformers.utils import logging +from .configuration_blip import BlipTextConfig + + +logger = logging.get_logger(__name__) + + +class BlipTextEmbeddings(nn.Module): + """Construct the embeddings from word and position embeddings.""" + + def __init__(self, config): + super().__init__() + self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size) + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + # position_ids (1, len position emb) is contiguous in memory and exported when serialized + self.register_buffer("position_ids", torch.arange(config.max_position_embeddings).expand((1, -1))) + self.position_embedding_type = getattr(config, "position_embedding_type", "absolute") + + self.config = config + + def forward( + self, input_ids=None, position_ids=None, inputs_embeds=None, past_key_values_length=0 + ): + if input_ids is not None: + input_shape = input_ids.size() + else: + input_shape = inputs_embeds.size()[:-1] + + seq_length = input_shape[1] + + if position_ids is None: + position_ids = self.position_ids[:, past_key_values_length : seq_length + past_key_values_length] + + if inputs_embeds is None: + inputs_embeds = self.word_embeddings(input_ids) + + embeddings = inputs_embeds + + if self.position_embedding_type == "absolute": + position_embeddings = self.position_embeddings(position_ids) + embeddings += position_embeddings + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + return embeddings + + +class BlipTextSelfAttention(nn.Module): + def __init__(self, config, is_cross_attention): + super().__init__() + self.config = config + if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"): + raise ValueError( + "The hidden size (%d) is not a multiple of the number of attention " + "heads (%d)" % (config.hidden_size, config.num_attention_heads) + ) + + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + if is_cross_attention: + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + else: + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + self.position_embedding_type = getattr(config, "position_embedding_type", "absolute") + if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query": + self.max_position_embeddings = config.max_position_embeddings + self.distance_embedding = nn.Embedding(2 * config.max_position_embeddings - 1, self.attention_head_size) + self.save_attention = False + + def save_attn_gradients(self, attn_gradients): + self.attn_gradients = attn_gradients + + def get_attn_gradients(self): + return self.attn_gradients + + def save_attention_map(self, attention_map): + self.attention_map = attention_map + + def get_attention_map(self): + return self.attention_map + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + mixed_query_layer = self.query(hidden_states) + + # If this is instantiated as a cross-attention module, the keys + # and values come from an encoder; the attention mask needs to be + # such that the encoder's padding tokens are not attended to. + is_cross_attention = encoder_hidden_states is not None + + if is_cross_attention: + key_layer = self.transpose_for_scores(self.key(encoder_hidden_states)) + value_layer = self.transpose_for_scores(self.value(encoder_hidden_states)) + attention_mask = encoder_attention_mask + elif past_key_value is not None: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + key_layer = torch.cat([past_key_value[0], key_layer], dim=2) + value_layer = torch.cat([past_key_value[1], value_layer], dim=2) + else: + key_layer = self.transpose_for_scores(self.key(hidden_states)) + value_layer = self.transpose_for_scores(self.value(hidden_states)) + + query_layer = self.transpose_for_scores(mixed_query_layer) + + past_key_value = (key_layer, value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + + if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query": + seq_length = hidden_states.size()[1] + position_ids_l = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(-1, 1) + position_ids_r = torch.arange(seq_length, dtype=torch.long, device=hidden_states.device).view(1, -1) + distance = position_ids_l - position_ids_r + positional_embedding = self.distance_embedding(distance + self.max_position_embeddings - 1) + positional_embedding = positional_embedding.to(dtype=query_layer.dtype) # fp16 compatibility + + if self.position_embedding_type == "relative_key": + relative_position_scores = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores + elif self.position_embedding_type == "relative_key_query": + relative_position_scores_query = torch.einsum("bhld,lrd->bhlr", query_layer, positional_embedding) + relative_position_scores_key = torch.einsum("bhrd,lrd->bhlr", key_layer, positional_embedding) + attention_scores = attention_scores + relative_position_scores_query + relative_position_scores_key + + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BlipTextModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + if is_cross_attention and self.save_attention: + self.save_attention_map(attention_probs) + attention_probs.register_hook(self.save_attn_gradients) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs_dropped = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs_dropped = attention_probs_dropped * head_mask + + context_layer = torch.matmul(attention_probs_dropped, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) + context_layer = context_layer.view(*new_context_layer_shape) + + outputs = (context_layer, attention_probs) if output_attentions else (context_layer,) + + outputs = outputs + (past_key_value,) + return outputs + + +class BlipTextSelfOutput(nn.Module): + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class BlipTextAttention(nn.Module): + def __init__(self, config, is_cross_attention=False): + super().__init__() + self.self = BlipTextSelfAttention(config, is_cross_attention) + self.output = BlipTextSelfOutput(config) + self.pruned_heads = set() + + def prune_heads(self, heads): + if len(heads) == 0: + return + heads, index = find_pruneable_heads_and_indices( + heads, self.self.num_attention_heads, self.self.attention_head_size, self.pruned_heads + ) + + # Prune linear layers + self.self.query = prune_linear_layer(self.self.query, index) + self.self.key = prune_linear_layer(self.self.key, index) + self.self.value = prune_linear_layer(self.self.value, index) + self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.self.num_attention_heads = self.self.num_attention_heads - len(heads) + self.self.all_head_size = self.self.attention_head_size * self.self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + ): + self_outputs = self.self( + hidden_states, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + ) + attention_output = self.output(self_outputs[0], hidden_states) + outputs = (attention_output,) + self_outputs[1:] # add attentions if we output them + return outputs + + +class BlipTextIntermediate(nn.Module): + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.intermediate_size) + if isinstance(config.hidden_act, str): + self.intermediate_act_fn = ACT2FN[config.hidden_act] + else: + self.intermediate_act_fn = config.hidden_act + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.intermediate_act_fn(hidden_states) + return hidden_states + + +class BlipTextOutput(nn.Module): + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.intermediate_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, hidden_states, input_tensor): + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class BlipTextLayer(nn.Module): + def __init__(self, config, layer_num): + super().__init__() + self.config = config + self.chunk_size_feed_forward = config.chunk_size_feed_forward + self.seq_len_dim = 1 + self.attention = BlipTextAttention(config) + self.layer_num = layer_num + if self.config.add_cross_attention: + self.crossattention = BlipTextAttention(config, is_cross_attention=self.config.add_cross_attention) + self.intermediate = BlipTextIntermediate(config) + self.output = BlipTextOutput(config) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + mode=None, + ): + # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 + self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None + self_attention_outputs = self.attention( + hidden_states, + attention_mask, + head_mask, + output_attentions=output_attentions, + past_key_value=self_attn_past_key_value, + ) + attention_output = self_attention_outputs[0] + + outputs = self_attention_outputs[1:-1] + present_key_value = self_attention_outputs[-1] + + if mode=='multimodal': + if encoder_hidden_states is None: + raise ValueError("encoder_hidden_states must be given for cross-attention layers") + + cross_attention_outputs = self.crossattention( + attention_output, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + output_attentions=output_attentions, + ) + attention_output = cross_attention_outputs[0] + outputs = outputs + cross_attention_outputs[1:-1] # add cross attentions if we output attention weights + layer_output = apply_chunking_to_forward( + self.feed_forward_chunk, self.chunk_size_feed_forward, self.seq_len_dim, attention_output + ) + outputs = (layer_output,) + outputs + + outputs = outputs + (present_key_value,) + + return outputs + + def feed_forward_chunk(self, attention_output): + intermediate_output = self.intermediate(attention_output) + layer_output = self.output(intermediate_output, attention_output) + return layer_output + + +class BlipTextEncoder(nn.Module): + def __init__(self, config): + super().__init__() + self.config = config + self.layer = nn.ModuleList([BlipTextLayer(config,i) for i in range(config.num_hidden_layers)]) + self.gradient_checkpointing = False + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=False, + output_hidden_states=False, + return_dict=True, + mode='multimodal', + ): + all_hidden_states = () if output_hidden_states else None + all_self_attentions = () if output_attentions else None + all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None + + next_decoder_cache = () if use_cache else None + + for i in range(self.config.num_hidden_layers): + layer_module = self.layer[i] + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + layer_head_mask = head_mask[i] if head_mask is not None else None + past_key_value = past_key_values[i] if past_key_values is not None else None + + if self.gradient_checkpointing and self.training: + + if use_cache: + logger.warn( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + ) + use_cache = False + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs, past_key_value, output_attentions) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer_module), + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + mode=mode, + ) + else: + layer_outputs = layer_module( + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + mode=mode, + ) + + hidden_states = layer_outputs[0] + if use_cache: + next_decoder_cache += (layer_outputs[-1],) + if output_attentions: + all_self_attentions = all_self_attentions + (layer_outputs[1],) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + if not return_dict: + return tuple( + v + for v in [ + hidden_states, + next_decoder_cache, + all_hidden_states, + all_self_attentions, + all_cross_attentions, + ] + if v is not None + ) + return BaseModelOutputWithPastAndCrossAttentions( + last_hidden_state=hidden_states, + past_key_values=next_decoder_cache, + hidden_states=all_hidden_states, + attentions=all_self_attentions, + cross_attentions=all_cross_attentions, + ) + + +class BlipTextPooler(nn.Module): + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.activation = nn.Tanh() + + def forward(self, hidden_states): + # We "pool" the model by simply taking the hidden state corresponding + # to the first token. + first_token_tensor = hidden_states[:, 0] + pooled_output = self.dense(first_token_tensor) + pooled_output = self.activation(pooled_output) + return pooled_output + + +class BlipTextPredictionHeadTransform(nn.Module): + def __init__(self, config): + super().__init__() + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + if isinstance(config.hidden_act, str): + self.transform_act_fn = ACT2FN[config.hidden_act] + else: + self.transform_act_fn = config.hidden_act + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.transform_act_fn(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + return hidden_states + + +class BlipTextLMPredictionHead(nn.Module): + def __init__(self, config): + super().__init__() + self.transform = BlipTextPredictionHeadTransform(config) + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False) + + self.bias = nn.Parameter(torch.zeros(config.vocab_size)) + + # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings` + self.decoder.bias = self.bias + + def forward(self, hidden_states): + hidden_states = self.transform(hidden_states) + hidden_states = self.decoder(hidden_states) + return hidden_states + + +class BlipTextOnlyMLMHead(nn.Module): + def __init__(self, config): + super().__init__() + self.predictions = BlipTextLMPredictionHead(config) + + def forward(self, sequence_output): + prediction_scores = self.predictions(sequence_output) + return prediction_scores + + +class BlipTextPreTrainedModel(PreTrainedModel): + """ + An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained + models. + """ + + config_class = BlipTextConfig + base_model_prefix = "bert" + _keys_to_ignore_on_load_missing = [r"position_ids"] + + def _init_weights(self, module): + """ Initialize the weights """ + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + + +class BlipTextModel(BlipTextPreTrainedModel): + """ + The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of + cross-attention is added between the self-attention layers, following the architecture described in `Attention is + all you need `__ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, + Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. + argument and :obj:`add_cross_attention` set to :obj:`True`; an :obj:`encoder_hidden_states` is then expected as an + input to the forward pass. + """ + + def __init__(self, config, add_pooling_layer=True): + super().__init__(config) + self.config = config + + self.embeddings = BlipTextEmbeddings(config) + + self.encoder = BlipTextEncoder(config) + + self.pooler = BlipTextPooler(config) if add_pooling_layer else None + + self.init_weights() + + + def get_input_embeddings(self): + return self.embeddings.word_embeddings + + def set_input_embeddings(self, value): + self.embeddings.word_embeddings = value + + def _prune_heads(self, heads_to_prune): + """ + Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base + class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + + def get_extended_attention_mask(self, attention_mask: Tensor, input_shape: Tuple[int], device: device, is_decoder: bool) -> Tensor: + """ + Makes broadcastable attention and causal masks so that future and masked tokens are ignored. + + Arguments: + attention_mask (:obj:`torch.Tensor`): + Mask with ones indicating tokens to attend to, zeros for tokens to ignore. + input_shape (:obj:`Tuple[int]`): + The shape of the input to the model. + device: (:obj:`torch.device`): + The device of the input to the model. + + Returns: + :obj:`torch.Tensor` The extended attention mask, with a the same dtype as :obj:`attention_mask.dtype`. + """ + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + if attention_mask.dim() == 3: + extended_attention_mask = attention_mask[:, None, :, :] + elif attention_mask.dim() == 2: + # Provided a padding mask of dimensions [batch_size, seq_length] + # - if the model is a decoder, apply a causal mask in addition to the padding mask + # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length] + if is_decoder: + batch_size, seq_length = input_shape + + seq_ids = torch.arange(seq_length, device=device) + causal_mask = seq_ids[None, None, :].repeat(batch_size, seq_length, 1) <= seq_ids[None, :, None] + # in case past_key_values are used we need to add a prefix ones mask to the causal mask + # causal and attention masks must have same type with pytorch version < 1.3 + causal_mask = causal_mask.to(attention_mask.dtype) + + if causal_mask.shape[1] < attention_mask.shape[1]: + prefix_seq_len = attention_mask.shape[1] - causal_mask.shape[1] + causal_mask = torch.cat( + [ + torch.ones((batch_size, seq_length, prefix_seq_len), device=device, dtype=causal_mask.dtype), + causal_mask, + ], + axis=-1, + ) + + extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :] + else: + extended_attention_mask = attention_mask[:, None, None, :] + else: + raise ValueError( + "Wrong shape for input_ids (shape {}) or attention_mask (shape {})".format( + input_shape, attention_mask.shape + ) + ) + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + extended_attention_mask = extended_attention_mask.to(dtype=self.dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + return extended_attention_mask + + def forward( + self, + input_ids=None, + attention_mask=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + is_decoder=False, + mode='multimodal', + ): + r""" + encoder_hidden_states (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if + the model is configured as a decoder. + encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in + the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``: + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. + If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids` + (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)` + instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`. + use_cache (:obj:`bool`, `optional`): + If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up + decoding (see :obj:`past_key_values`). + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if is_decoder: + use_cache = use_cache if use_cache is not None else self.config.use_cache + else: + use_cache = False + + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = input_ids.size() + batch_size, seq_length = input_shape + device = input_ids.device + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + batch_size, seq_length = input_shape + device = inputs_embeds.device + elif encoder_embeds is not None: + input_shape = encoder_embeds.size()[:-1] + batch_size, seq_length = input_shape + device = encoder_embeds.device + else: + raise ValueError("You have to specify either input_ids or inputs_embeds or encoder_embeds") + + # past_key_values_length + past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0 + + if attention_mask is None: + attention_mask = torch.ones(((batch_size, seq_length + past_key_values_length)), device=device) + + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + extended_attention_mask: torch.Tensor = self.get_extended_attention_mask(attention_mask, input_shape, + device, is_decoder) + + # If a 2D or 3D attention mask is provided for the cross-attention + # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length] + if encoder_hidden_states is not None: + if type(encoder_hidden_states) == list: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states[0].size() + else: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size() + encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length) + + if type(encoder_attention_mask) == list: + encoder_extended_attention_mask = [self.invert_attention_mask(mask) for mask in encoder_attention_mask] + elif encoder_attention_mask is None: + encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device) + encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask) + else: + encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask) + else: + encoder_extended_attention_mask = None + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers) + + if encoder_embeds is None: + embedding_output = self.embeddings( + input_ids=input_ids, + position_ids=position_ids, + inputs_embeds=inputs_embeds, + past_key_values_length=past_key_values_length, + ) + else: + embedding_output = encoder_embeds + + encoder_outputs = self.encoder( + embedding_output, + attention_mask=extended_attention_mask, + head_mask=head_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + mode=mode, + ) + sequence_output = encoder_outputs[0] + pooled_output = self.pooler(sequence_output) if self.pooler is not None else None + + if not return_dict: + return (sequence_output, pooled_output) + encoder_outputs[1:] + + return BaseModelOutputWithPoolingAndCrossAttentions( + last_hidden_state=sequence_output, + pooler_output=pooled_output, + past_key_values=encoder_outputs.past_key_values, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + cross_attentions=encoder_outputs.cross_attentions, + ) + + + +class BlipTextLMHeadModel(BlipTextPreTrainedModel): + + _keys_to_ignore_on_load_unexpected = [r"pooler"] + _keys_to_ignore_on_load_missing = [r"position_ids", r"predictions.decoder.bias"] + + def __init__(self, config): + super().__init__(config) + + self.bert = BlipTextModel(config, add_pooling_layer=False) + self.cls = BlipTextOnlyMLMHead(config) + + self.init_weights() + + def get_output_embeddings(self): + return self.cls.predictions.decoder + + def set_output_embeddings(self, new_embeddings): + self.cls.predictions.decoder = new_embeddings + + def forward( + self, + input_ids=None, + attention_mask=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + labels=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + return_logits=False, + is_decoder=True, + reduction='mean', + mode='multimodal', + ): + r""" + encoder_hidden_states (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if + the model is configured as a decoder. + encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in + the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``: + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in + ``[-100, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) Tokens with indices set to ``-100`` are + ignored (masked), the loss is only computed for the tokens with labels n ``[0, ..., config.vocab_size]`` + past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): + Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. + If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids` + (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)` + instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`. + use_cache (:obj:`bool`, `optional`): + If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up + decoding (see :obj:`past_key_values`). + Returns: + Example:: + >>> from transformers import BlipTextTokenizer, BlipTextLMHeadModel, BlipTextConfig + >>> import torch + >>> tokenizer = BlipTextTokenizer.from_pretrained('bert-base-cased') + >>> config = BlipTextConfig.from_pretrained("bert-base-cased") + >>> model = BlipTextLMHeadModel.from_pretrained('bert-base-cased', config=config) + >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") + >>> outputs = model(**inputs) + >>> prediction_logits = outputs.logits + """ + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + if labels is not None: + use_cache = False + + outputs = self.bert( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + is_decoder=is_decoder, + mode=mode, + ) + + sequence_output = outputs[0] + prediction_scores = self.cls(sequence_output) + + if return_logits: + return prediction_scores[:, :-1, :].contiguous() + + lm_loss = None + if labels is not None: + # we are doing next-token prediction; shift prediction scores and input ids by one + shifted_prediction_scores = prediction_scores[:, :-1, :].contiguous() + labels = labels[:, 1:].contiguous() + loss_fct = CrossEntropyLoss(reduction=reduction, label_smoothing=0.1) + lm_loss = loss_fct(shifted_prediction_scores.view(-1, self.config.vocab_size), labels.view(-1)) + if reduction=='none': + lm_loss = lm_loss.view(prediction_scores.size(0),-1).sum(1) + + if not return_dict: + output = (prediction_scores,) + outputs[2:] + return ((lm_loss,) + output) if lm_loss is not None else output + + return CausalLMOutputWithCrossAttentions( + loss=lm_loss, + logits=prediction_scores, + past_key_values=outputs.past_key_values, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + cross_attentions=outputs.cross_attentions, + ) + + def prepare_inputs_for_generation(self, input_ids, past=None, attention_mask=None, **model_kwargs): + input_shape = input_ids.shape + # if model is used as a decoder in encoder-decoder model, the decoder attention mask is created on the fly + if attention_mask is None: + attention_mask = input_ids.new_ones(input_shape) + + # cut decoder_input_ids if past is used + if past is not None: + input_ids = input_ids[:, -1:] + + return { + "input_ids": input_ids, + "attention_mask": attention_mask, + "past_key_values": past, + "encoder_hidden_states": model_kwargs.get("encoder_hidden_states", None), + "encoder_attention_mask": model_kwargs.get("encoder_attention_mask", None), + "is_decoder": True, + } + + def _reorder_cache(self, past, beam_idx): + reordered_past = () + for layer_past in past: + reordered_past += (tuple(past_state.index_select(0, beam_idx) for past_state in layer_past),) + return reordered_past From 955d7b59865f61381eee49d3c9c2dd195146a030 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Fri, 9 Dec 2022 16:01:02 +0000 Subject: [PATCH 07/96] fix --- docs/source/en/index.mdx | 2 +- .../models/bert/configuration_bert.py | 2 - src/transformers/models/bert/modeling_bert.py | 12 +- .../models/blip/configuration_blip.py | 16 +- src/transformers/models/blip/modeling_blip.py | 47 +++-- .../models/blip/modeling_blip_text.py | 165 +++++++++--------- src/transformers/utils/dummy_pt_objects.py | 24 +++ 7 files changed, 145 insertions(+), 123 deletions(-) diff --git a/docs/source/en/index.mdx b/docs/source/en/index.mdx index 4df907b5702f..dbd9a5ac090f 100644 --- a/docs/source/en/index.mdx +++ b/docs/source/en/index.mdx @@ -237,7 +237,7 @@ Flax), PyTorch, and/or TensorFlow. | BiT | ❌ | ❌ | ✅ | ❌ | ❌ | | Blenderbot | ✅ | ✅ | ✅ | ✅ | ✅ | | BlenderbotSmall | ✅ | ✅ | ✅ | ✅ | ✅ | -| BLIP | ❌ | ❌ | ❌ | ❌ | ❌ | +| BLIP | ❌ | ❌ | ✅ | ❌ | ❌ | | BLOOM | ❌ | ✅ | ✅ | ❌ | ❌ | | CamemBERT | ✅ | ✅ | ✅ | ✅ | ❌ | | CANINE | ✅ | ❌ | ✅ | ❌ | ❌ | diff --git a/src/transformers/models/bert/configuration_bert.py b/src/transformers/models/bert/configuration_bert.py index 146175263ecd..b2d64b7fde67 100644 --- a/src/transformers/models/bert/configuration_bert.py +++ b/src/transformers/models/bert/configuration_bert.py @@ -156,7 +156,6 @@ def __init__( position_embedding_type="absolute", use_cache=True, classifier_dropout=None, - use_token_type_embed=True, **kwargs ): super().__init__(pad_token_id=pad_token_id, **kwargs) @@ -176,7 +175,6 @@ def __init__( self.position_embedding_type = position_embedding_type self.use_cache = use_cache self.classifier_dropout = classifier_dropout - self.use_token_type_embed = use_token_type_embed class BertOnnxConfig(OnnxConfig): diff --git a/src/transformers/models/bert/modeling_bert.py b/src/transformers/models/bert/modeling_bert.py index 7e143466ffe9..16abb6c871ac 100755 --- a/src/transformers/models/bert/modeling_bert.py +++ b/src/transformers/models/bert/modeling_bert.py @@ -185,9 +185,7 @@ def __init__(self, config): super().__init__() self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id) self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size) - self.token_type_embeddings = ( - nn.Embedding(config.type_vocab_size, config.hidden_size) if config.use_token_type_embed else None - ) + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.hidden_size) # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load # any TensorFlow checkpoint file @@ -231,13 +229,9 @@ def forward( if inputs_embeds is None: inputs_embeds = self.word_embeddings(input_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) - if self.token_type_embeddings is not None: - token_type_embeddings = self.token_type_embeddings(token_type_ids) - embeddings = inputs_embeds + token_type_embeddings - else: - embeddings = inputs_embeds - + embeddings = inputs_embeds + token_type_embeddings if self.position_embedding_type == "absolute": position_embeddings = self.position_embeddings(position_ids) embeddings += position_embeddings diff --git a/src/transformers/models/blip/configuration_blip.py b/src/transformers/models/blip/configuration_blip.py index 7bc5ed83e6a9..25f33774bf88 100644 --- a/src/transformers/models/blip/configuration_blip.py +++ b/src/transformers/models/blip/configuration_blip.py @@ -57,8 +57,8 @@ class BlipTextConfig(PretrainedConfig): just in case (e.g., 512 or 1024 or 2048). hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`): The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, - `"relu"`, `"selu"` and `"gelu_new"` ``"gelu"` are supported. layer_norm_eps (`float`, *optional*, - defaults to 1e-5): The epsilon used by the layer normalization layers. + `"relu"`, `"selu"` and `"gelu_new"` ``"gelu"` are supported. layer_norm_eps (`float`, *optional*, defaults + to 1e-5): The epsilon used by the layer normalization layers. attention_dropout (`float`, *optional*, defaults to 0.0): The dropout ratio for the attention probabilities. dropout (`float`, *optional*, defaults to 0.0): @@ -111,7 +111,13 @@ def __init__( use_cache=True, **kwargs ): - super().__init__(pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, sep_token_id=sep_token_id, **kwargs) + super().__init__( + pad_token_id=pad_token_id, + bos_token_id=bos_token_id, + eos_token_id=eos_token_id, + sep_token_id=sep_token_id, + **kwargs, + ) self.vocab_size = vocab_size self.type_vocab_size = type_vocab_size @@ -176,8 +182,8 @@ class BlipVisionConfig(PretrainedConfig): The size (resolution) of each patch. hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`): The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`, - `"relu"`, `"selu"` and `"gelu_new"` ``"gelu"` are supported. layer_norm_eps (`float`, *optional*, - defaults to 1e-5): The epsilon used by the layer normalization layers. + `"relu"`, `"selu"` and `"gelu_new"` ``"gelu"` are supported. layer_norm_eps (`float`, *optional*, defaults + to 1e-5): The epsilon used by the layer normalization layers. dropout (`float`, *optional*, defaults to 0.0): The dropout probabilitiy for all fully connected layers in the embeddings, encoder, and pooler. attention_dropout (`float`, *optional*, defaults to 0.0): diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 3b429c82afc3..f8cd0e166265 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. """ PyTorch BLIP model.""" -import math - from dataclasses import dataclass from typing import Any, Optional, Tuple, Union @@ -22,7 +20,6 @@ import torch.utils.checkpoint from torch import nn - from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import PreTrainedModel @@ -196,7 +193,7 @@ def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: class_embeds = self.class_embedding.expand(batch_size, 1, -1) embeddings = torch.cat([class_embeds, patch_embeds], dim=1) - embeddings = embeddings + self.position_embedding[:, :embeddings.size(1), :] + embeddings = embeddings + self.position_embedding[:, : embeddings.size(1), :] return embeddings @@ -249,12 +246,12 @@ def __init__(self, config): self.dropout = nn.Dropout(config.attention_dropout) self.qkv = nn.Linear(self.embed_dim, 3 * self.embed_dim) - + self.proj = nn.Linear(self.embed_dim, self.embed_dim) def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() - + def _split_heads(self, fused_qkv: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """ Split the last dimension into (num_heads, head_dim) without making any copies, results share same memory @@ -283,8 +280,16 @@ def forward( mixed_qkv = self.qkv(hidden_states) # query_states, key_states, value_states = self._split_heads(mixed_qkv) - qkv = self.qkv(hidden_states).reshape(bsz, tgt_len, 3, self.num_heads, embed_dim // self.num_heads).permute(2, 0, 3, 1, 4) - query_states, key_states, value_states = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple) + mixed_qkv = ( + self.qkv(hidden_states) + .reshape(bsz, tgt_len, 3, self.num_heads, embed_dim // self.num_heads) + .permute(2, 0, 3, 1, 4) + ) + query_states, key_states, value_states = ( + mixed_qkv[0], + mixed_qkv[1], + mixed_qkv[2], + ) # make torchscript happy (cannot use tensor as tuple) # Take the dot product between "query" and "key" to get the raw attention scores. attention_scores = torch.matmul(query_states, key_states.transpose(-1, -2)) @@ -401,7 +406,6 @@ def _init_weights(self, module): elif isinstance(module, BlipAttention): factor = self.config.initializer_factor in_proj_std = (module.embed_dim**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor - out_proj_std = (module.embed_dim**-0.5) * factor nn.init.normal_(module.qkv.weight, std=in_proj_std) elif isinstance(module, BlipMLP): factor = self.config.initializer_factor @@ -1277,7 +1281,7 @@ def forward( return_dict=return_dict, ) - image_embeds = vision_outputs[0] + image_embeds = vision_outputs[0] if input_ids is not None: # Case 1: image captioning @@ -1300,25 +1304,21 @@ def forward( attentions=vision_outputs.attentions, ) - def generate(self, - input_ids: torch.LongTensor, - pixel_values: torch.FloatTensor, - **generate_kwargs - ): + def generate(self, input_ids: torch.LongTensor, pixel_values: torch.FloatTensor, **generate_kwargs): r""" - Overrides `generate` function to be able to use the model as a conditional generator + Overrides `generate` function to be able to use the model as a conditional generator - Args: - `input_ids` + Args: + `input_ids` """ vision_outputs = self.vision_model( pixel_values=pixel_values, ) - image_embeds = vision_outputs[0] + image_embeds = vision_outputs[0] - image_atts = torch.ones(image_embeds.size()[:-1],dtype=torch.long).to(image_embeds.device) - model_kwargs = {"encoder_hidden_states": image_embeds, "encoder_attention_mask":image_atts} + image_atts = torch.ones(image_embeds.size()[:-1], dtype=torch.long).to(image_embeds.device) + model_kwargs = {"encoder_hidden_states": image_embeds, "encoder_attention_mask": image_atts} if isinstance(input_ids, list): input_ids = torch.LongTensor(input_ids) @@ -1328,10 +1328,9 @@ def generate(self, outputs = self.text_decoder.generate( input_ids=input_ids[:, :-1], eos_token_id=self.config.text_config.sep_token_id, - pad_token_id=self.config.text_config.pad_token_id, - **generate_kwargs, + pad_token_id=self.config.text_config.pad_token_id, + **generate_kwargs, **model_kwargs, ) return outputs - diff --git a/src/transformers/models/blip/modeling_blip_text.py b/src/transformers/models/blip/modeling_blip_text.py index 8145d25ace6f..51de996deaf1 100644 --- a/src/transformers/models/blip/modeling_blip_text.py +++ b/src/transformers/models/blip/modeling_blip_text.py @@ -1,4 +1,4 @@ -''' +""" * Copyright (c) 2022, salesforce.com, inc. * All rights reserved. * SPDX-License-Identifier: BSD-3-Clause @@ -6,19 +6,17 @@ * By Junnan Li * Based on huggingface code base * https://github.com/huggingface/transformers/blob/v4.15.0/src/transformers/models/bert -''' +""" import math from typing import Tuple import torch -from torch import Tensor, device, nn import torch.utils.checkpoint -from torch import nn +from torch import Tensor, device, nn from torch.nn import CrossEntropyLoss from transformers.activations import ACT2FN - from transformers.modeling_outputs import ( BaseModelOutputWithPastAndCrossAttentions, BaseModelOutputWithPoolingAndCrossAttentions, @@ -31,6 +29,7 @@ prune_linear_layer, ) from transformers.utils import logging + from .configuration_blip import BlipTextConfig @@ -53,12 +52,10 @@ def __init__(self, config): # position_ids (1, len position emb) is contiguous in memory and exported when serialized self.register_buffer("position_ids", torch.arange(config.max_position_embeddings).expand((1, -1))) self.position_embedding_type = getattr(config, "position_embedding_type", "absolute") - + self.config = config - def forward( - self, input_ids=None, position_ids=None, inputs_embeds=None, past_key_values_length=0 - ): + def forward(self, input_ids=None, position_ids=None, inputs_embeds=None, past_key_values_length=0): if input_ids is not None: input_shape = input_ids.size() else: @@ -88,10 +85,10 @@ def __init__(self, config, is_cross_attention): self.config = config if config.hidden_size % config.num_attention_heads != 0 and not hasattr(config, "embedding_size"): raise ValueError( - "The hidden size (%d) is not a multiple of the number of attention " - "heads (%d)" % (config.hidden_size, config.num_attention_heads) + "The hidden size (%d) is not a multiple of the number of attention heads (%d)" + % (config.hidden_size, config.num_attention_heads) ) - + self.num_attention_heads = config.num_attention_heads self.attention_head_size = int(config.hidden_size / config.num_attention_heads) self.all_head_size = self.num_attention_heads * self.attention_head_size @@ -109,20 +106,20 @@ def __init__(self, config, is_cross_attention): if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query": self.max_position_embeddings = config.max_position_embeddings self.distance_embedding = nn.Embedding(2 * config.max_position_embeddings - 1, self.attention_head_size) - self.save_attention = False - + self.save_attention = False + def save_attn_gradients(self, attn_gradients): self.attn_gradients = attn_gradients - + def get_attn_gradients(self): return self.attn_gradients - + def save_attention_map(self, attention_map): self.attention_map = attention_map - + def get_attention_map(self): return self.attention_map - + def transpose_for_scores(self, x): new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size) x = x.view(*new_x_shape) @@ -188,10 +185,10 @@ def forward( # Normalize the attention scores to probabilities. attention_probs = nn.Softmax(dim=-1)(attention_scores) - + if is_cross_attention and self.save_attention: self.save_attention_map(attention_probs) - attention_probs.register_hook(self.save_attn_gradients) + attention_probs.register_hook(self.save_attn_gradients) # This is actually dropping out entire tokens to attend to, which might # seem a bit unusual, but is taken from the original Transformer paper. @@ -311,8 +308,8 @@ def __init__(self, config, layer_num): self.config = config self.chunk_size_feed_forward = config.chunk_size_feed_forward self.seq_len_dim = 1 - self.attention = BlipTextAttention(config) - self.layer_num = layer_num + self.attention = BlipTextAttention(config) + self.layer_num = layer_num if self.config.add_cross_attention: self.crossattention = BlipTextAttention(config, is_cross_attention=self.config.add_cross_attention) self.intermediate = BlipTextIntermediate(config) @@ -343,8 +340,8 @@ def forward( outputs = self_attention_outputs[1:-1] present_key_value = self_attention_outputs[-1] - if mode=='multimodal': - if encoder_hidden_states is None: + if mode == "multimodal": + if encoder_hidden_states is None: raise ValueError("encoder_hidden_states must be given for cross-attention layers") cross_attention_outputs = self.crossattention( @@ -356,7 +353,7 @@ def forward( output_attentions=output_attentions, ) attention_output = cross_attention_outputs[0] - outputs = outputs + cross_attention_outputs[1:-1] # add cross attentions if we output attention weights + outputs = outputs + cross_attention_outputs[1:-1] # add cross attentions if we output attention weights layer_output = apply_chunking_to_forward( self.feed_forward_chunk, self.chunk_size_feed_forward, self.seq_len_dim, attention_output ) @@ -376,7 +373,7 @@ class BlipTextEncoder(nn.Module): def __init__(self, config): super().__init__() self.config = config - self.layer = nn.ModuleList([BlipTextLayer(config,i) for i in range(config.num_hidden_layers)]) + self.layer = nn.ModuleList([BlipTextLayer(config, i) for i in range(config.num_hidden_layers)]) self.gradient_checkpointing = False def forward( @@ -391,14 +388,14 @@ def forward( output_attentions=False, output_hidden_states=False, return_dict=True, - mode='multimodal', + mode="multimodal", ): all_hidden_states = () if output_hidden_states else None all_self_attentions = () if output_attentions else None all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None next_decoder_cache = () if use_cache else None - + for i in range(self.config.num_hidden_layers): layer_module = self.layer[i] if output_hidden_states: @@ -545,7 +542,7 @@ class BlipTextPreTrainedModel(PreTrainedModel): _keys_to_ignore_on_load_missing = [r"position_ids"] def _init_weights(self, module): - """ Initialize the weights """ + """Initialize the weights""" if isinstance(module, (nn.Linear, nn.Embedding)): # Slightly different from the TF version which uses truncated_normal for initialization # cf https://github.com/pytorch/pytorch/pull/5617 @@ -562,9 +559,8 @@ class BlipTextModel(BlipTextPreTrainedModel): The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in `Attention is all you need `__ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, - Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. - argument and :obj:`add_cross_attention` set to :obj:`True`; an :obj:`encoder_hidden_states` is then expected as an - input to the forward pass. + Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. argument and :obj:`add_cross_attention` set to + :obj:`True`; an :obj:`encoder_hidden_states` is then expected as an input to the forward pass. """ def __init__(self, config, add_pooling_layer=True): @@ -572,13 +568,12 @@ def __init__(self, config, add_pooling_layer=True): self.config = config self.embeddings = BlipTextEmbeddings(config) - + self.encoder = BlipTextEncoder(config) self.pooler = BlipTextPooler(config) if add_pooling_layer else None self.init_weights() - def get_input_embeddings(self): return self.embeddings.word_embeddings @@ -594,8 +589,9 @@ class PreTrainedModel for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - - def get_extended_attention_mask(self, attention_mask: Tensor, input_shape: Tuple[int], device: device, is_decoder: bool) -> Tensor: + def get_extended_attention_mask( + self, attention_mask: Tensor, input_shape: Tuple[int], device: device, is_decoder: bool + ) -> Tensor: """ Makes broadcastable attention and causal masks so that future and masked tokens are ignored. @@ -626,16 +622,18 @@ def get_extended_attention_mask(self, attention_mask: Tensor, input_shape: Tuple # in case past_key_values are used we need to add a prefix ones mask to the causal mask # causal and attention masks must have same type with pytorch version < 1.3 causal_mask = causal_mask.to(attention_mask.dtype) - + if causal_mask.shape[1] < attention_mask.shape[1]: prefix_seq_len = attention_mask.shape[1] - causal_mask.shape[1] causal_mask = torch.cat( [ - torch.ones((batch_size, seq_length, prefix_seq_len), device=device, dtype=causal_mask.dtype), + torch.ones( + (batch_size, seq_length, prefix_seq_len), device=device, dtype=causal_mask.dtype + ), causal_mask, ], axis=-1, - ) + ) extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :] else: @@ -655,7 +653,7 @@ def get_extended_attention_mask(self, attention_mask: Tensor, input_shape: Tuple extended_attention_mask = extended_attention_mask.to(dtype=self.dtype) # fp16 compatibility extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 return extended_attention_mask - + def forward( self, input_ids=None, @@ -672,20 +670,23 @@ def forward( output_hidden_states=None, return_dict=None, is_decoder=False, - mode='multimodal', + mode="multimodal", ): r""" - encoder_hidden_states (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. + encoder_hidden_states (: + obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`): Sequence + of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model + is configured as a decoder. encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. - past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids` + past_key_values (: + obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of + shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): Contains precomputed key + and value hidden states of the attention blocks. Can be used to speed up decoding. If + :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids` (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)` instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`. use_cache (:obj:`bool`, `optional`): @@ -713,9 +714,9 @@ def forward( input_shape = inputs_embeds.size()[:-1] batch_size, seq_length = input_shape device = inputs_embeds.device - elif encoder_embeds is not None: + elif encoder_embeds is not None: input_shape = encoder_embeds.size()[:-1] - batch_size, seq_length = input_shape + batch_size, seq_length = input_shape device = encoder_embeds.device else: raise ValueError("You have to specify either input_ids or inputs_embeds or encoder_embeds") @@ -725,11 +726,12 @@ def forward( if attention_mask is None: attention_mask = torch.ones(((batch_size, seq_length + past_key_values_length)), device=device) - + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] # ourselves in which case we just need to make it broadcastable to all heads. - extended_attention_mask: torch.Tensor = self.get_extended_attention_mask(attention_mask, input_shape, - device, is_decoder) + extended_attention_mask: torch.Tensor = self.get_extended_attention_mask( + attention_mask, input_shape, device, is_decoder + ) # If a 2D or 3D attention mask is provided for the cross-attention # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length] @@ -739,13 +741,13 @@ def forward( else: encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size() encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length) - + if type(encoder_attention_mask) == list: encoder_extended_attention_mask = [self.invert_attention_mask(mask) for mask in encoder_attention_mask] elif encoder_attention_mask is None: encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device) encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask) - else: + else: encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask) else: encoder_extended_attention_mask = None @@ -756,7 +758,7 @@ def forward( # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] head_mask = self.get_head_mask(head_mask, self.config.num_hidden_layers) - + if encoder_embeds is None: embedding_output = self.embeddings( input_ids=input_ids, @@ -766,7 +768,7 @@ def forward( ) else: embedding_output = encoder_embeds - + encoder_outputs = self.encoder( embedding_output, attention_mask=extended_attention_mask, @@ -796,7 +798,6 @@ def forward( ) - class BlipTextLMHeadModel(BlipTextPreTrainedModel): _keys_to_ignore_on_load_unexpected = [r"pooler"] @@ -831,15 +832,16 @@ def forward( output_attentions=None, output_hidden_states=None, return_dict=None, - return_logits=False, + return_logits=False, is_decoder=True, - reduction='mean', - mode='multimodal', + reduction="mean", + mode="multimodal", ): r""" - encoder_hidden_states (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`): - Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if - the model is configured as a decoder. + encoder_hidden_states (: + obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`): Sequence + of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model + is configured as a decoder. encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``: @@ -849,9 +851,11 @@ def forward( Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in ``[-100, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) Tokens with indices set to ``-100`` are ignored (masked), the loss is only computed for the tokens with labels n ``[0, ..., config.vocab_size]`` - past_key_values (:obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): - Contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding. - If :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids` + past_key_values (: + obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of + shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): Contains precomputed key + and value hidden states of the attention blocks. Can be used to speed up decoding. If + :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids` (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)` instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`. use_cache (:obj:`bool`, `optional`): @@ -859,14 +863,11 @@ def forward( decoding (see :obj:`past_key_values`). Returns: Example:: - >>> from transformers import BlipTextTokenizer, BlipTextLMHeadModel, BlipTextConfig - >>> import torch - >>> tokenizer = BlipTextTokenizer.from_pretrained('bert-base-cased') - >>> config = BlipTextConfig.from_pretrained("bert-base-cased") - >>> model = BlipTextLMHeadModel.from_pretrained('bert-base-cased', config=config) - >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") - >>> outputs = model(**inputs) - >>> prediction_logits = outputs.logits + >>> from transformers import BlipTextTokenizer, BlipTextLMHeadModel, BlipTextConfig >>> import torch >>> + tokenizer = BlipTextTokenizer.from_pretrained('bert-base-cased') >>> config = + BlipTextConfig.from_pretrained("bert-base-cased") >>> model = + BlipTextLMHeadModel.from_pretrained('bert-base-cased', config=config) >>> inputs = tokenizer("Hello, my dog + is cute", return_tensors="pt") >>> outputs = model(**inputs) >>> prediction_logits = outputs.logits """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict if labels is not None: @@ -888,22 +889,22 @@ def forward( is_decoder=is_decoder, mode=mode, ) - + sequence_output = outputs[0] prediction_scores = self.cls(sequence_output) - + if return_logits: - return prediction_scores[:, :-1, :].contiguous() + return prediction_scores[:, :-1, :].contiguous() lm_loss = None if labels is not None: # we are doing next-token prediction; shift prediction scores and input ids by one shifted_prediction_scores = prediction_scores[:, :-1, :].contiguous() labels = labels[:, 1:].contiguous() - loss_fct = CrossEntropyLoss(reduction=reduction, label_smoothing=0.1) + loss_fct = CrossEntropyLoss(reduction=reduction, label_smoothing=0.1) lm_loss = loss_fct(shifted_prediction_scores.view(-1, self.config.vocab_size), labels.view(-1)) - if reduction=='none': - lm_loss = lm_loss.view(prediction_scores.size(0),-1).sum(1) + if reduction == "none": + lm_loss = lm_loss.view(prediction_scores.size(0), -1).sum(1) if not return_dict: output = (prediction_scores,) + outputs[2:] @@ -929,8 +930,8 @@ def prepare_inputs_for_generation(self, input_ids, past=None, attention_mask=Non input_ids = input_ids[:, -1:] return { - "input_ids": input_ids, - "attention_mask": attention_mask, + "input_ids": input_ids, + "attention_mask": attention_mask, "past_key_values": past, "encoder_hidden_states": model_kwargs.get("encoder_hidden_states", None), "encoder_attention_mask": model_kwargs.get("encoder_attention_mask", None), diff --git a/src/transformers/utils/dummy_pt_objects.py b/src/transformers/utils/dummy_pt_objects.py index af230ce82f39..6e9616e3688a 100644 --- a/src/transformers/utils/dummy_pt_objects.py +++ b/src/transformers/utils/dummy_pt_objects.py @@ -1112,6 +1112,30 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["torch"]) +BLIP_PRETRAINED_MODEL_ARCHIVE_LIST = None + + +class BlipForConditionalGeneration(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class BlipModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class BlipPreTrainedModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + BLOOM_PRETRAINED_MODEL_ARCHIVE_LIST = None From 05f18495776f53d9069d30bcd1f68dcde5b5d2d7 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Fri, 9 Dec 2022 16:34:42 +0000 Subject: [PATCH 08/96] add docstring --- src/transformers/models/blip/modeling_blip.py | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index f8cd0e166265..6e7011134899 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -1304,12 +1304,37 @@ def forward( attentions=vision_outputs.attentions, ) - def generate(self, input_ids: torch.LongTensor, pixel_values: torch.FloatTensor, **generate_kwargs): + @torch.no_grad() + def generate( + self, input_ids: torch.LongTensor, pixel_values: torch.FloatTensor, **generate_kwargs + ) -> torch.LongTensor: r""" Overrides `generate` function to be able to use the model as a conditional generator - Args: - `input_ids` + Parameters: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + The sequence used as a prompt for the generation. + pixel_values (`torch.FloatTensor` of shape `(batch_size, image_width, image_height)`: + Input image to be processed + + + Examples: + ```python + >>> from PIL import Image + >>> import requests + >>> from transformers import BlipTokenizer, BlipForImageCaptioning + + >>> model = BLIPForImageCaptioning.from_pretrained("ybelkada/blip-base") + >>> processor = CLIPProcessor.from_pretrained("ybelkada/blip-base") + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> inputs = processor(images=image, return_tensors="pt") + + >>> outputs = model(**inputs) + >>> image_embeds = outputs.image_embeds + ``` """ vision_outputs = self.vision_model( pixel_values=pixel_values, From 12ffea9aa103586742d6215a7ae7cfcfd8fd157e Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Sun, 11 Dec 2022 12:08:11 +0000 Subject: [PATCH 09/96] CI tests pass --- src/transformers/__init__.py | 8 + src/transformers/models/auto/modeling_auto.py | 2 +- src/transformers/models/blip/__init__.py | 10 +- .../models/blip/configuration_blip.py | 3 +- src/transformers/models/blip/modeling_blip.py | 644 ++++++++++-------- .../models/blip/modeling_blip_text.py | 16 +- tests/models/blip/test_modeling_blip.py | 421 +++++++----- 7 files changed, 601 insertions(+), 503 deletions(-) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index ec956f27437f..597fb193462e 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -1098,6 +1098,10 @@ "BlipForConditionalGeneration", "BlipModel", "BlipPreTrainedModel", + "BlipForVisualQuestionAnswering", + "BlipTextModel", + "BlipVisionModel", + "BlipForImageTextRetrieval", ] ) _import_structure["models.bloom"].extend( @@ -4248,8 +4252,12 @@ from .models.blip import ( BLIP_PRETRAINED_MODEL_ARCHIVE_LIST, BlipForConditionalGeneration, + BlipForVisualQuestionAnswering, BlipModel, + BlipTextModel, + BlipVisionModel, BlipPreTrainedModel, + BlipForImageTextRetrieval, ) from .models.bloom import ( BLOOM_PRETRAINED_MODEL_ARCHIVE_LIST, diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index 91198c073e47..465e42b1fc83 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -40,7 +40,7 @@ ("bit", "BitModel"), ("blenderbot", "BlenderbotModel"), ("blenderbot-small", "BlenderbotSmallModel"), - ("blip", "BLIPModel"), + ("blip", "BlipModel"), ("bloom", "BloomModel"), ("camembert", "CamembertModel"), ("canine", "CanineModel"), diff --git a/src/transformers/models/blip/__init__.py b/src/transformers/models/blip/__init__.py index 95973da089fd..8f28687f205a 100644 --- a/src/transformers/models/blip/__init__.py +++ b/src/transformers/models/blip/__init__.py @@ -37,9 +37,13 @@ else: _import_structure["modeling_blip"] = [ "BLIP_PRETRAINED_MODEL_ARCHIVE_LIST", - "BlipPModel", + "BlipModel", "BlipPreTrainedModel", "BlipForConditionalGeneration", + "BlipForVisualQuestionAnswering", + "BlipVisionModel", + "BlipTextModel", + "BlipForImageTextRetrieval", ] if TYPE_CHECKING: @@ -54,7 +58,11 @@ from .modeling_blip import ( BLIP_PRETRAINED_MODEL_ARCHIVE_LIST, BlipForConditionalGeneration, + BlipForVisualQuestionAnswering, BlipModel, + BlipVisionModel, + BlipTextModel, + BlipForImageTextRetrieval, BlipPreTrainedModel, ) diff --git a/src/transformers/models/blip/configuration_blip.py b/src/transformers/models/blip/configuration_blip.py index 25f33774bf88..32464107176c 100644 --- a/src/transformers/models/blip/configuration_blip.py +++ b/src/transformers/models/blip/configuration_blip.py @@ -313,7 +313,7 @@ class BlipConfig(PretrainedConfig): is_composition = True def __init__( - self, text_config=None, vision_config=None, projection_dim=512, logit_scale_init_value=2.6592, **kwargs + self, text_config=None, vision_config=None, projection_dim=512, logit_scale_init_value=2.6592, image_text_hidden_size=256,**kwargs ): super().__init__(**kwargs) @@ -339,6 +339,7 @@ def __init__( self.projection_dim = projection_dim self.logit_scale_init_value = logit_scale_init_value self.initializer_factor = 1.0 + self.image_text_hidden_size = image_text_hidden_size @classmethod def from_text_vision_configs(cls, text_config: BlipTextConfig, vision_config: BlipVisionConfig, **kwargs): diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 6e7011134899..0bfad9436491 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -19,9 +19,10 @@ import torch import torch.utils.checkpoint from torch import nn +import torch.nn.functional as F from ...activations import ACT2FN -from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, CausalLMOutputWithCrossAttentions from ...modeling_utils import PreTrainedModel from ...utils import ( ModelOutput, @@ -31,7 +32,7 @@ replace_return_docstrings, ) from .configuration_blip import BlipConfig, BlipTextConfig, BlipVisionConfig -from .modeling_blip_text import BlipTextLMHeadModel +from .modeling_blip_text import BlipTextLMHeadModel, BlipTextModel logger = logging.get_logger(__name__) @@ -42,23 +43,6 @@ "ybelkada/blip-base", # See all BLIP models at https://huggingface.co/models?filter=blip ] - - -# Copied from transformers.models.bart.modeling_bart._expand_mask -def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): - """ - Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. - """ - bsz, src_len = mask.size() - tgt_len = tgt_len if tgt_len is not None else src_len - - expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype) - - inverted_mask = 1.0 - expanded_mask - - return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min) - - # contrastive loss function, adapted from # https://sachinruk.github.io/blog/pytorch/pytorch%20lightning/loss%20function/gpu/2021/03/07/BLIP.html def contrastive_loss(logits: torch.Tensor) -> torch.Tensor: @@ -71,6 +55,36 @@ def blip_loss(similarity: torch.Tensor) -> torch.Tensor: image_loss = contrastive_loss(similarity.t()) return (caption_loss + image_loss) / 2.0 +@dataclass +# Copied from transformers.models.clip.modeling_clip.CLIPVisionModelOutput with CLIP->BLIP +class BlipForConditionalGenerationModelOutput(ModelOutput): + """ + Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states. + + Args: + image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`): + The image embeddings obtained by applying the projection layer to the pooler_output. + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. + + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + """ + loss: Optional[Tuple[torch.FloatTensor]] = None + decoder_logits: Optional[Tuple[torch.FloatTensor]] = None + image_embeds: Optional[torch.FloatTensor] = None + last_hidden_state: torch.FloatTensor = None + hidden_states: Optional[Tuple[torch.FloatTensor]] = None + attentions: Optional[Tuple[torch.FloatTensor]] = None + @dataclass # Copied from transformers.models.clip.modeling_clip.CLIPVisionModelOutput with CLIP->BLIP @@ -95,7 +109,7 @@ class BlipVisionModelOutput(ModelOutput): Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. """ - + loss: Optional[torch.FloatTensor] = None image_embeds: Optional[torch.FloatTensor] = None last_hidden_state: torch.FloatTensor = None hidden_states: Optional[Tuple[torch.FloatTensor]] = None @@ -144,13 +158,13 @@ class BlipOutput(ModelOutput): The scaled dot product scores between `text_embeds` and `image_embeds`. This represents the text-image similarity scores. text_embeds(`torch.FloatTensor` of shape `(batch_size, output_dim`): - The text embeddings obtained by applying the projection layer to the pooled output of [`BLIPTextModel`]. + The text embeddings obtained by applying the projection layer to the pooled output of [`BlipTextModel`]. image_embeds(`torch.FloatTensor` of shape `(batch_size, output_dim`): - The image embeddings obtained by applying the projection layer to the pooled output of [`BLIPVisionModel`]. + The image embeddings obtained by applying the projection layer to the pooled output of [`BlipVisionModel`]. text_model_output(`BaseModelOutputWithPooling`): - The output of the [`BLIPTextModel`]. + The output of the [`BlipTextModel`]. vision_model_output(`BaseModelOutputWithPooling`): - The output of the [`BLIPVisionModel`]. + The output of the [`BlipVisionModel`]. """ loss: Optional[torch.FloatTensor] = None @@ -197,6 +211,7 @@ def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: return embeddings + class BlipTextEmbeddings(nn.Module): def __init__(self, config: BlipTextConfig): super().__init__() @@ -400,13 +415,18 @@ def _init_weights(self, module): module.token_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02) module.position_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02) elif isinstance(module, BlipVisionEmbeddings): - factor = self.config.initializer_factor - nn.init.normal_(module.class_embedding, mean=0.0, std=module.embed_dim**-0.5 * factor) - nn.init.normal_(module.patch_embedding.weight, std=module.config.initializer_range * factor) + module.class_embedding.data.normal_(mean=0.0, std=factor * 0.02) + module.patch_embedding.weight.data.normal_(mean=0.0, std=module.config.initializer_range * factor) + module.patch_embedding.bias.data.normal_(mean=0.0, std=module.config.initializer_range * factor) elif isinstance(module, BlipAttention): factor = self.config.initializer_factor in_proj_std = (module.embed_dim**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor nn.init.normal_(module.qkv.weight, std=in_proj_std) + if module.qkv.bias is not None: + nn.init.normal_(module.qkv.bias, std=in_proj_std) + nn.init.normal_(module.proj.weight, std=in_proj_std) + if module.proj.bias is not None: + nn.init.normal_(module.proj.bias, std=in_proj_std) elif isinstance(module, BlipMLP): factor = self.config.initializer_factor in_proj_std = ( @@ -424,16 +444,15 @@ def _init_weights(self, module): module.visual_projection.weight, std=module.vision_embed_dim**-0.5 * self.config.initializer_factor, ) - elif isinstance(module, BLIPTextModelWithProjection): - nn.init.normal_( - module.text_projection.weight, - std=self.config.hidden_size**-0.5 * self.config.initializer_factor, - ) + elif isinstance(module, nn.Conv2d) or isinstance(module, nn.Embedding) or isinstance(module, nn.Linear): + nn.init.normal_(module.weight, mean=0.0, std=factor * 0.02) + if hasattr(module, "bias") and module.bias is not None: + nn.init.normal_(module.bias, mean=0.0, std=factor * 0.02) - if isinstance(module, nn.LayerNorm): + elif isinstance(module, nn.LayerNorm): module.bias.data.zero_() module.weight.data.fill_(1.0) - if isinstance(module, nn.Linear) and module.bias is not None: + elif isinstance(module, nn.Linear) and module.bias is not None: module.bias.data.zero_() def _set_gradient_checkpointing(self, module, value=False): @@ -619,7 +638,6 @@ def custom_forward(*inputs): create_custom_forward(encoder_layer), hidden_states, attention_mask, - causal_attention_mask, ) else: layer_outputs = encoder_layer( @@ -643,158 +661,11 @@ def custom_forward(*inputs): ) -class BLIPTextTransformer(nn.Module): - def __init__(self, config: BlipTextConfig): - super().__init__() - self.config = config - embed_dim = config.hidden_size - self.embeddings = BlipTextEmbeddings(config) - self.encoder = BlipEncoder(config) - self.final_layer_norm = nn.LayerNorm(embed_dim) - - @add_start_docstrings_to_model_forward(BLIP_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=BlipTextConfig) - def forward( - self, - input_ids: Optional[torch.Tensor] = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.Tensor] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - ) -> Union[Tuple, BaseModelOutputWithPooling]: - r""" - Returns: - - """ - output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions - output_hidden_states = ( - output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states - ) - return_dict = return_dict if return_dict is not None else self.config.use_return_dict - - if input_ids is None: - raise ValueError("You have to specify either input_ids") - - input_shape = input_ids.size() - input_ids = input_ids.view(-1, input_shape[-1]) - - hidden_states = self.embeddings(input_ids=input_ids, position_ids=position_ids) - - bsz, seq_len = input_shape - # BLIP's text model uses causal mask, prepare it here. - # https://github.com/openai/BLIP/blob/cfcffb90e69f37bf2ff1e988237a0fbe41f33c04/blip/model.py#L324 - causal_attention_mask = self._build_causal_attention_mask(bsz, seq_len, hidden_states.dtype).to( - hidden_states.device - ) - # expand attention_mask - if attention_mask is not None: - # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] - attention_mask = _expand_mask(attention_mask, hidden_states.dtype) - - encoder_outputs = self.encoder( - inputs_embeds=hidden_states, - attention_mask=attention_mask, - causal_attention_mask=causal_attention_mask, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - ) - - last_hidden_state = encoder_outputs[0] - last_hidden_state = self.final_layer_norm(last_hidden_state) - - # text_embeds.shape = [batch_size, sequence_length, transformer.width] - # take features from the eot embedding (eot_token is the highest number in each sequence) - # casting to torch.int for onnx compatibility: argmax doesn't support int64 inputs with opset 14 - pooled_output = last_hidden_state[ - torch.arange(last_hidden_state.shape[0], device=input_ids.device), input_ids.to(torch.int).argmax(dim=-1) - ] - - if not return_dict: - return (last_hidden_state, pooled_output) + encoder_outputs[1:] - - return BaseModelOutputWithPooling( - last_hidden_state=last_hidden_state, - pooler_output=pooled_output, - hidden_states=encoder_outputs.hidden_states, - attentions=encoder_outputs.attentions, - ) - - def _build_causal_attention_mask(self, bsz, seq_len, dtype): - # lazily create causal attention mask, with full attention between the vision tokens - # pytorch uses additive attention mask; fill with -inf - mask = torch.empty(bsz, seq_len, seq_len, dtype=dtype) - mask.fill_(torch.tensor(torch.finfo(dtype).min)) - mask.triu_(1) # zero out the lower diagonal - mask = mask.unsqueeze(1) # expand mask - return mask - - -@add_start_docstrings( - """The text model from BLIP without any head or projection on top.""", - BLIP_START_DOCSTRING, -) -class BLIPTextModel(BlipPreTrainedModel): - config_class = BlipTextConfig - - _no_split_modules = ["BlipEncoderLayer"] - - def __init__(self, config: BlipTextConfig): - super().__init__(config) - self.text_model = BLIPTextTransformer(config) - # Initialize weights and apply final processing - self.post_init() - - def get_input_embeddings(self) -> nn.Module: - return self.text_model.embeddings.token_embedding - - def set_input_embeddings(self, value): - self.text_model.embeddings.token_embedding = value - - @add_start_docstrings_to_model_forward(BLIP_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=BlipTextConfig) - def forward( - self, - input_ids: Optional[torch.Tensor] = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.Tensor] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - ) -> Union[Tuple, BaseModelOutputWithPooling]: - r""" - Returns: - - Examples: - - ```python - >>> from transformers import CLIPTokenizer, BLIPTextModel - - >>> model = BLIPTextModel.from_pretrained("ybelkada/blip-base") - >>> tokenizer = CLIPTokenizer.from_pretrained("ybelkada/blip-base") - - >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") - - >>> outputs = model(**inputs) - >>> last_hidden_state = outputs.last_hidden_state - >>> pooled_output = outputs.pooler_output # pooled (EOS token) states - ```""" - return_dict = return_dict if return_dict is not None else self.config.use_return_dict - - return self.text_model( - input_ids=input_ids, - attention_mask=attention_mask, - position_ids=position_ids, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - ) - +class BlipVisionModel(BlipPreTrainedModel): + main_input_name = "pixel_values" -class BlipVisionTransformer(nn.Module): def __init__(self, config: BlipVisionConfig): - super().__init__() + super().__init__(config) self.config = config embed_dim = config.hidden_size @@ -802,6 +673,8 @@ def __init__(self, config: BlipVisionConfig): self.encoder = BlipEncoder(config) self.post_layernorm = nn.LayerNorm(embed_dim) + self.post_init() + @add_start_docstrings_to_model_forward(BLIP_VISION_INPUTS_DOCSTRING) @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=BlipVisionConfig) def forward( @@ -848,64 +721,9 @@ def forward( hidden_states=encoder_outputs.hidden_states, attentions=encoder_outputs.attentions, ) - - -@add_start_docstrings( - """The vision model from BLIP without any head or projection on top.""", - BLIP_START_DOCSTRING, -) -class BLIPVisionModel(BlipPreTrainedModel): - config_class = BlipVisionConfig - main_input_name = "pixel_values" - - def __init__(self, config: BlipVisionConfig): - super().__init__(config) - self.vision_model = BlipVisionTransformer(config) - # Initialize weights and apply final processing - self.post_init() - - def get_input_embeddings(self) -> nn.Module: - return self.vision_model.embeddings.patch_embedding - - @add_start_docstrings_to_model_forward(BLIP_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=BlipVisionConfig) - def forward( - self, - pixel_values: Optional[torch.FloatTensor] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, - return_dict: Optional[bool] = None, - ) -> Union[Tuple, BaseModelOutputWithPooling]: - r""" - Returns: - - Examples: - - ```python - >>> from PIL import Image - >>> import requests - >>> from transformers import CLIPProcessor, BLIPVisionModel - - >>> model = BLIPVisionModel.from_pretrained("ybelkada/blip-base") - >>> processor = CLIPProcessor.from_pretrained("ybelkada/blip-base") - - >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" - >>> image = Image.open(requests.get(url, stream=True).raw) - - >>> inputs = processor(images=image, return_tensors="pt") - - >>> outputs = model(**inputs) - >>> last_hidden_state = outputs.last_hidden_state - >>> pooled_output = outputs.pooler_output # pooled CLS states - ```""" - return_dict = return_dict if return_dict is not None else self.config.use_return_dict - - return self.vision_model( - pixel_values=pixel_values, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, - return_dict=return_dict, - ) + + def get_input_embeddings(self): + return self.embeddings @add_start_docstrings(BLIP_START_DOCSTRING) @@ -934,8 +752,8 @@ def __init__(self, config: BlipConfig): self.text_embed_dim = text_config.hidden_size self.vision_embed_dim = vision_config.hidden_size - self.text_model = BLIPTextTransformer(text_config) - self.vision_model = BlipVisionTransformer(vision_config) + self.text_model = BlipTextModel(text_config) + self.vision_model = BlipVisionModel(vision_config) self.visual_projection = nn.Linear(self.vision_embed_dim, self.projection_dim, bias=False) self.text_projection = nn.Linear(self.text_embed_dim, self.projection_dim, bias=False) @@ -957,7 +775,7 @@ def get_text_features( r""" Returns: text_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The text embeddings obtained by - applying the projection layer to the pooled output of [`BLIPTextModel`]. + applying the projection layer to the pooled output of [`BlipTextModel`]. Examples: @@ -1002,7 +820,7 @@ def get_image_features( r""" Returns: image_features (`torch.FloatTensor` of shape `(batch_size, output_dim`): The image embeddings obtained by - applying the projection layer to the pooled output of [`BLIPVisionModel`]. + applying the projection layer to the pooled output of [`BlipVisionModel`]. Examples: @@ -1136,84 +954,151 @@ def forward( @add_start_docstrings( """ - BLIP Text Model with a projection layer on top (a linear layer on top of the pooled output). + BLIP Vision Model with a projection layer on top (a linear layer on top of the pooled output). """, BLIP_START_DOCSTRING, ) -class BLIPTextModelWithProjection(BlipPreTrainedModel): - config_class = BlipTextConfig - - _no_split_modules = ["BlipEncoderLayer"] +class BlipForConditionalGeneration(BlipPreTrainedModel): + config_class = BlipConfig + _keys_to_ignore_on_load_missing = [r"text_decoder.cls.predictions.decoder.bias"] - def __init__(self, config: BlipTextConfig): + def __init__(self, config: BlipConfig): super().__init__(config) - self.text_model = BLIPTextTransformer(config) + self.vision_model = BlipVisionModel(config.vision_config) - self.text_projection = nn.Linear(config.hidden_size, config.projection_dim, bias=False) + self.text_decoder = BlipTextLMHeadModel(config.text_config) + + self.decoder_input_ids = config.text_config.bos_token_id # Initialize weights and apply final processing self.post_init() def get_input_embeddings(self) -> nn.Module: - return self.text_model.embeddings.token_embedding - - def set_input_embeddings(self, value): - self.text_model.embeddings.token_embedding = value + return self.vision_model.embeddings.patch_embedding - @add_start_docstrings_to_model_forward(BLIP_TEXT_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BlipTextModelOutput, config_class=BlipTextConfig) + @add_start_docstrings_to_model_forward(BLIP_VISION_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=BlipVisionModelOutput, config_class=BlipVisionConfig) def forward( self, - input_ids: Optional[torch.Tensor] = None, - attention_mask: Optional[torch.Tensor] = None, - position_ids: Optional[torch.Tensor] = None, + input_ids: torch.LongTensor, + pixel_values: torch.FloatTensor, + attention_mask: Optional[torch.LongTensor] = None, output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, BlipTextModelOutput]: + ) -> Union[Tuple, BlipVisionModelOutput]: r""" Returns: Examples: ```python - >>> from transformers import CLIPTokenizer, BLIPTextModelWithProjection + >>> from PIL import Image + >>> import requests + >>> from transformers import CLIPProcessor, BLIPForImageCaptioning - >>> model = BLIPTextModelWithProjection.from_pretrained("ybelkada/blip-base") - >>> tokenizer = CLIPTokenizer.from_pretrained("ybelkada/blip-base") + >>> model = BLIPForImageCaptioning.from_pretrained("ybelkada/blip-base") + >>> processor = CLIPProcessor.from_pretrained("ybelkada/blip-base") - >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> inputs = processor(images=image, return_tensors="pt") >>> outputs = model(**inputs) - >>> text_embeds = outputs.text_embeds + >>> image_embeds = outputs.image_embeds ```""" return_dict = return_dict if return_dict is not None else self.config.use_return_dict - text_outputs = self.text_model( - input_ids=input_ids, - attention_mask=attention_mask, - position_ids=position_ids, + vision_outputs = self.vision_model( + pixel_values=pixel_values, output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, ) - pooled_output = text_outputs[1] + image_embeds = vision_outputs[0] - text_embeds = self.text_projection(pooled_output) + decoder_targets = input_ids.masked_fill(input_ids == self.decoder_input_ids, -100) + + outputs = self.text_decoder( + input_ids=input_ids, + attention_mask=attention_mask, + encoder_hidden_states=image_embeds, + labels=decoder_targets, + return_dict=return_dict, + ) if not return_dict: - outputs = (text_embeds, text_outputs[0]) + text_outputs[2:] + outputs = (outputs[0], outputs[1], image_embeds, vision_outputs[0]) + vision_outputs[2:] return tuple(output for output in outputs if output is not None) - return BlipTextModelOutput( - text_embeds=text_embeds, - last_hidden_state=text_outputs.last_hidden_state, - hidden_states=text_outputs.hidden_states, - attentions=text_outputs.attentions, + return BlipForConditionalGenerationModelOutput( + loss=outputs.loss, + decoder_logits=outputs.logits, + image_embeds=image_embeds, + last_hidden_state=vision_outputs.last_hidden_state, + hidden_states=vision_outputs.hidden_states, + attentions=vision_outputs.attentions, + ) + + @torch.no_grad() + def generate( + self, input_ids: torch.LongTensor, pixel_values: torch.FloatTensor, **generate_kwargs + ) -> torch.LongTensor: + r""" + Overrides `generate` function to be able to use the model as a conditional generator + + Parameters: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + The sequence used as a prompt for the generation. + pixel_values (`torch.FloatTensor` of shape `(batch_size, image_width, image_height)`: + Input image to be processed + + + Examples: + ```python + >>> from PIL import Image + >>> import requests + >>> from transformers import BlipTokenizer, BlipForImageCaptioning + + >>> model = BLIPForImageCaptioning.from_pretrained("ybelkada/blip-base") + >>> processor = CLIPProcessor.from_pretrained("ybelkada/blip-base") + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> inputs = processor(images=image, return_tensors="pt") + + >>> outputs = model(**inputs) + >>> image_embeds = outputs.image_embeds + ``` + """ + vision_outputs = self.vision_model( + pixel_values=pixel_values, ) + image_embeds = vision_outputs[0] + + image_atts = torch.ones(image_embeds.size()[:-1], dtype=torch.long).to(image_embeds.device) + model_kwargs = {"encoder_hidden_states": image_embeds, "encoder_attention_mask": image_atts} + + if isinstance(input_ids, list): + input_ids = torch.LongTensor(input_ids) + + input_ids[:, 0] = self.config.text_config.bos_token_id + + outputs = self.text_decoder.generate( + input_ids=input_ids[:, :-1], + eos_token_id=self.config.text_config.sep_token_id, + pad_token_id=self.config.text_config.pad_token_id, + **generate_kwargs, + **model_kwargs, + ) + + return outputs + @add_start_docstrings( """ @@ -1221,18 +1106,22 @@ def forward( """, BLIP_START_DOCSTRING, ) -class BlipForConditionalGeneration(BlipPreTrainedModel): - config_class = BlipVisionConfig - main_input_name = "pixel_values" +class BlipForVisualQuestionAnswering(BlipPreTrainedModel): + config_class = BlipConfig + _keys_to_ignore_on_load_missing = [r"text_decoder.cls.predictions.decoder.bias"] + def __init__(self, config: BlipConfig): super().__init__(config) - self.vision_model = BlipVisionTransformer(config.vision_config) + self.vision_model = BlipVisionModel(config.vision_config) + + self.text_encoder = BlipTextModel(config.text_config, add_pooling_layer=False) self.text_decoder = BlipTextLMHeadModel(config.text_config) - self.decoder_input_ids = config.text_config.bos_token_id + self.decoder_pad_token_id = config.text_config.pad_token_id + self.decoder_bos_token_id = config.text_config.bos_token_id # Initialize weights and apply final processing self.post_init() @@ -1246,9 +1135,11 @@ def forward( self, input_ids: torch.LongTensor, pixel_values: torch.FloatTensor, + answer_input_ids: Optional[torch.LongTensor] = None, + answer_attention_mask: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.LongTensor] = None, output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, - decoder_input_ids: Optional[int] = 0, return_dict: Optional[bool] = None, ) -> Union[Tuple, BlipVisionModelOutput]: r""" @@ -1273,6 +1164,7 @@ def forward( >>> image_embeds = outputs.image_embeds ```""" return_dict = return_dict if return_dict is not None else self.config.use_return_dict + batch_size, _ = input_ids.shape vision_outputs = self.vision_model( pixel_values=pixel_values, @@ -1282,31 +1174,54 @@ def forward( ) image_embeds = vision_outputs[0] + image_atts = torch.ones(image_embeds.size()[:-1], dtype=torch.long) + + question_embeds = self.text_encoder( + input_ids=input_ids, + attention_mask=attention_mask, + encoder_hidden_states=image_embeds, + encoder_attention_mask=image_atts, + return_dict=return_dict, + ) - if input_ids is not None: - # Case 1: image captioning - decoder_targets = input_ids.masked_fill(input_ids == self.decoder_input_ids, -100) + question_embeds = question_embeds[0] if not return_dict else question_embeds.last_hidden_state + + if answer_input_ids is None: + answer_input_ids = torch.LongTensor([self.decoder_bos_token_id]).repeat((batch_size, 1)) + answer_targets = answer_input_ids.masked_fill(answer_input_ids == self.decoder_pad_token_id, -100) - outputs = self.text_decoder( - input_ids, - encoder_hidden_states=image_embeds, - labels=decoder_targets, - ) + answer_output = self.text_decoder( + input_ids=answer_input_ids, + attention_mask=answer_attention_mask, + encoder_hidden_states=question_embeds, + encoder_attention_mask=attention_mask, + labels=answer_targets, + return_dict=return_dict, + reduction='none', + ) + + decoder_loss = answer_output.loss.mean() if return_dict else answer_output[0].mean() if not return_dict: - outputs = (image_embeds, vision_outputs[0]) + vision_outputs[2:] + outputs = (decoder_loss, image_embeds, vision_outputs[0]) + vision_outputs[2:] return tuple(output for output in outputs if output is not None) return BlipVisionModelOutput( + loss=decoder_loss, image_embeds=image_embeds, last_hidden_state=vision_outputs.last_hidden_state, hidden_states=vision_outputs.hidden_states, attentions=vision_outputs.attentions, ) - + @torch.no_grad() def generate( - self, input_ids: torch.LongTensor, pixel_values: torch.FloatTensor, **generate_kwargs + self, + input_ids: torch.LongTensor, + pixel_values: torch.FloatTensor, + attention_mask: Optional[torch.LongTensor]=None, + use_rank_mode: Optional[bool]=False, + **generate_kwargs ) -> torch.LongTensor: r""" Overrides `generate` function to be able to use the model as a conditional generator @@ -1348,10 +1263,23 @@ def generate( if isinstance(input_ids, list): input_ids = torch.LongTensor(input_ids) - input_ids[:, 0] = self.config.text_config.bos_token_id + question_outputs = self.text_encoder( + input_ids=input_ids, + attention_mask=attention_mask, + encoder_hidden_states=image_embeds, + encoder_attention_mask=image_atts, + return_dict=False + ) + + question_embeds = question_outputs[0] + + question_atts = torch.ones(question_embeds.size()[:-1],dtype=torch.long).to(question_embeds.device) + model_kwargs = {"encoder_hidden_states": question_embeds, "encoder_attention_mask":question_atts} + + bos_ids = torch.full((question_embeds.size(0),1),fill_value=self.decoder_bos_token_id,device=question_embeds.device) outputs = self.text_decoder.generate( - input_ids=input_ids[:, :-1], + input_ids=bos_ids, eos_token_id=self.config.text_config.sep_token_id, pad_token_id=self.config.text_config.pad_token_id, **generate_kwargs, @@ -1359,3 +1287,117 @@ def generate( ) return outputs + +@add_start_docstrings( + """ + BLIP Vision Model with a projection layer on top (a linear layer on top of the pooled output). + """, + BLIP_START_DOCSTRING, +) +class BlipForImageTextRetrieval(BlipPreTrainedModel): + config_class = BlipConfig + + def __init__(self, config: BlipConfig): + super().__init__(config) + + self.vision_model = BlipVisionModel(config.vision_config) + + self.text_encoder = BlipTextModel(config.text_config, add_pooling_layer=False) + + self.vision_proj = nn.Linear(config.vision_config.hidden_size, config.image_text_hidden_size) + self.text_proj = nn.Linear(config.text_config.hidden_size, config.image_text_hidden_size) + + self.itm_head = nn.Linear(config.text_config.hidden_size, 2) + + self.decoder_pad_token_id = config.text_config.pad_token_id + self.decoder_bos_token_id = config.text_config.bos_token_id + + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self) -> nn.Module: + return self.vision_model.embeddings.patch_embedding + + @add_start_docstrings_to_model_forward(BLIP_VISION_INPUTS_DOCSTRING) + @replace_return_docstrings(output_type=BlipVisionModelOutput, config_class=BlipVisionConfig) + def forward( + self, + input_ids: torch.LongTensor, + pixel_values: torch.FloatTensor, + use_itm_head: Optional[bool] = True, + attention_mask: Optional[torch.LongTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BlipVisionModelOutput]: + r""" + Returns: + + Examples: + + ```python + >>> from PIL import Image + >>> import requests + >>> from transformers import CLIPProcessor, BLIPForImageCaptioning + + >>> model = BLIPForImageCaptioning.from_pretrained("ybelkada/blip-base") + >>> processor = CLIPProcessor.from_pretrained("ybelkada/blip-base") + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> image = Image.open(requests.get(url, stream=True).raw) + + >>> inputs = processor(images=image, return_tensors="pt") + + >>> outputs = model(**inputs) + >>> image_embeds = outputs.image_embeds + ```""" + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + batch_size, _ = input_ids.shape + + vision_outputs = self.vision_model( + pixel_values=pixel_values, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + image_embeds = vision_outputs[0] + image_atts = torch.ones(image_embeds.size()[:-1], dtype=torch.long) + + if use_itm_head: + question_embeds = self.text_encoder( + input_ids=input_ids, + attention_mask=attention_mask, + encoder_hidden_states=image_embeds, + encoder_attention_mask=image_atts, + return_dict=return_dict, + ) + question_embeds = question_embeds[0] if not return_dict else question_embeds.last_hidden_state + + output = self.itm_head(question_embeds[:,0,:]) + else: + question_embeds = self.text_encoder( + input_ids=input_ids, + attention_mask=attention_mask, + return_dict=return_dict, + mode="text", + ) + question_embeds = question_embeds[0] if not return_dict else question_embeds.last_hidden_state + + image_feat = F.normalize(self.vision_proj(image_embeds[:,0,:]),dim=-1) + text_feat = F.normalize(self.text_proj(question_embeds[:,0,:]),dim=-1) + + output = image_feat @ text_feat.t() + + + + if not return_dict: + outputs = (output, vision_outputs[0]) + vision_outputs[2:] + return tuple(output for output in outputs if output is not None) + + return BlipVisionModelOutput( + image_embeds=output, + last_hidden_state=vision_outputs.last_hidden_state, + hidden_states=vision_outputs.hidden_states, + attentions=vision_outputs.attentions, + ) \ No newline at end of file diff --git a/src/transformers/models/blip/modeling_blip_text.py b/src/transformers/models/blip/modeling_blip_text.py index 51de996deaf1..7dd7c9805ea2 100644 --- a/src/transformers/models/blip/modeling_blip_text.py +++ b/src/transformers/models/blip/modeling_blip_text.py @@ -67,6 +67,7 @@ def forward(self, input_ids=None, position_ids=None, inputs_embeds=None, past_ke position_ids = self.position_ids[:, past_key_values_length : seq_length + past_key_values_length] if inputs_embeds is None: + input_ids = input_ids.to(self.word_embeddings.weight.device) inputs_embeds = self.word_embeddings(input_ids) embeddings = inputs_embeds @@ -181,7 +182,7 @@ def forward( attention_scores = attention_scores / math.sqrt(self.attention_head_size) if attention_mask is not None: # Apply the attention mask is (precomputed for all layers in BlipTextModel forward() function) - attention_scores = attention_scores + attention_mask + attention_scores = attention_scores + attention_mask.to(attention_scores.device) # Normalize the attention scores to probabilities. attention_probs = nn.Softmax(dim=-1)(attention_scores) @@ -340,10 +341,7 @@ def forward( outputs = self_attention_outputs[1:-1] present_key_value = self_attention_outputs[-1] - if mode == "multimodal": - if encoder_hidden_states is None: - raise ValueError("encoder_hidden_states must be given for cross-attention layers") - + if encoder_hidden_states is not None: cross_attention_outputs = self.crossattention( attention_output, attention_mask, @@ -573,7 +571,7 @@ def __init__(self, config, add_pooling_layer=True): self.pooler = BlipTextPooler(config) if add_pooling_layer else None - self.init_weights() + self.post_init() def get_input_embeddings(self): return self.embeddings.word_embeddings @@ -725,7 +723,7 @@ def forward( past_key_values_length = past_key_values[0][0].shape[2] if past_key_values is not None else 0 if attention_mask is None: - attention_mask = torch.ones(((batch_size, seq_length + past_key_values_length)), device=device) + attention_mask = torch.ones(((batch_size, seq_length + past_key_values_length))) # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] # ourselves in which case we just need to make it broadcastable to all heads. @@ -809,8 +807,6 @@ def __init__(self, config): self.bert = BlipTextModel(config, add_pooling_layer=False) self.cls = BlipTextOnlyMLMHead(config) - self.init_weights() - def get_output_embeddings(self): return self.cls.predictions.decoder @@ -900,7 +896,7 @@ def forward( if labels is not None: # we are doing next-token prediction; shift prediction scores and input ids by one shifted_prediction_scores = prediction_scores[:, :-1, :].contiguous() - labels = labels[:, 1:].contiguous() + labels = labels[:, 1:].contiguous().to(shifted_prediction_scores.device) loss_fct = CrossEntropyLoss(reduction=reduction, label_smoothing=0.1) lm_loss = loss_fct(shifted_prediction_scores.view(-1, self.config.vocab_size), labels.view(-1)) if reduction == "none": diff --git a/tests/models/blip/test_modeling_blip.py b/tests/models/blip/test_modeling_blip.py index 38bc771d8f06..2807887bcfdb 100644 --- a/tests/models/blip/test_modeling_blip.py +++ b/tests/models/blip/test_modeling_blip.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" Testing suite for the PyTorch BLIP model. """ +""" Testing suite for the PyTorch Blip model. """ import inspect @@ -23,11 +23,8 @@ import numpy as np import requests -import transformers -from transformers import BLIPConfig, BLIPTextConfig, BLIPVisionConfig +from transformers import BlipConfig, BlipTextConfig, BlipVisionConfig from transformers.testing_utils import ( - is_flax_available, - is_pt_flax_cross_test, require_torch, require_vision, slow, @@ -50,11 +47,12 @@ from torch import nn from transformers import ( - BLIPModel, - BLIPTextModel, - BLIPTextModelWithProjection, - BLIPVisionModel, - BLIPVisionModelWithProjection, + BlipModel, + BlipTextModel, + BlipVisionModel, + BlipForConditionalGeneration, + BlipForVisualQuestionAnswering, + BlipForImageTextRetrieval, ) from transformers.models.blip.modeling_blip import BLIP_PRETRAINED_MODEL_ARCHIVE_LIST @@ -64,16 +62,7 @@ from transformers import CLIPProcessor - -if is_flax_available(): - import jax.numpy as jnp - from transformers.modeling_flax_pytorch_utils import ( - convert_pytorch_state_dict_to_flax, - load_flax_weights_in_pytorch_model, - ) - - -class BLIPVisionModelTester: +class BlipVisionModelTester: def __init__( self, parent, @@ -119,7 +108,7 @@ def prepare_config_and_inputs(self): return config, pixel_values def get_config(self): - return BLIPVisionConfig( + return BlipVisionConfig( image_size=self.image_size, patch_size=self.patch_size, num_channels=self.num_channels, @@ -134,7 +123,7 @@ def get_config(self): ) def create_and_check_model(self, config, pixel_values): - model = BLIPVisionModel(config=config) + model = BlipVisionModel(config=config) model.to(torch_device) model.eval() with torch.no_grad(): @@ -146,19 +135,6 @@ def create_and_check_model(self, config, pixel_values): self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, num_patches + 1, self.hidden_size)) self.parent.assertEqual(result.pooler_output.shape, (self.batch_size, self.hidden_size)) - def create_and_check_model_with_projection(self, config, pixel_values): - model = BLIPVisionModelWithProjection(config=config) - model.to(torch_device) - model.eval() - with torch.no_grad(): - result = model(pixel_values) - # expected sequence length = num_patches + 1 (we add 1 for the [CLS] token) - image_size = (self.image_size, self.image_size) - patch_size = (self.patch_size, self.patch_size) - num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0]) - self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, num_patches + 1, self.hidden_size)) - self.parent.assertEqual(result.image_embeds.shape, (self.batch_size, self.projection_dim)) - def prepare_config_and_inputs_for_common(self): config_and_inputs = self.prepare_config_and_inputs() config, pixel_values = config_and_inputs @@ -167,26 +143,26 @@ def prepare_config_and_inputs_for_common(self): @require_torch -class BLIPVisionModelTest(ModelTesterMixin, unittest.TestCase): +class BlipVisionModelTest(ModelTesterMixin, unittest.TestCase): """ - Here we also overwrite some of the tests of test_modeling_common.py, as BLIP does not use input_ids, inputs_embeds, + Here we also overwrite some of the tests of test_modeling_common.py, as Blip does not use input_ids, inputs_embeds, attention_mask and seq_length. """ - all_model_classes = (BLIPVisionModel, BLIPVisionModelWithProjection) if is_torch_available() else () + all_model_classes = (BlipVisionModel,) if is_torch_available() else () fx_compatible = False test_pruning = False test_resize_embeddings = False test_head_masking = False def setUp(self): - self.model_tester = BLIPVisionModelTester(self) - self.config_tester = ConfigTester(self, config_class=BLIPVisionConfig, has_text_modality=False, hidden_size=37) + self.model_tester = BlipVisionModelTester(self) + self.config_tester = ConfigTester(self, config_class=BlipVisionConfig, has_text_modality=False, hidden_size=37) def test_config(self): self.config_tester.run_common_tests() - @unittest.skip(reason="BLIP does not use inputs_embeds") + @unittest.skip(reason="Blip does not use inputs_embeds") def test_inputs_embeds(self): pass @@ -215,39 +191,28 @@ def test_model(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_model(*config_and_inputs) - def test_model_with_projection(self): - config_and_inputs = self.model_tester.prepare_config_and_inputs() - self.model_tester.create_and_check_model_with_projection(*config_and_inputs) - def test_training(self): pass def test_training_gradient_checkpointing(self): pass - @unittest.skip(reason="BLIPVisionModel has no base class and is not available in MODEL_MAPPING") + @unittest.skip(reason="BlipVisionModel has no base class and is not available in MODEL_MAPPING") def test_save_load_fast_init_from_base(self): pass - @unittest.skip(reason="BLIPVisionModel has no base class and is not available in MODEL_MAPPING") + @unittest.skip(reason="BlipVisionModel has no base class and is not available in MODEL_MAPPING") def test_save_load_fast_init_to_base(self): pass @slow def test_model_from_pretrained(self): for model_name in BLIP_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: - model = BLIPVisionModel.from_pretrained(model_name) + model = BlipVisionModel.from_pretrained(model_name) self.assertIsNotNone(model) - @slow - def test_model_with_projection_from_pretrained(self): - for model_name in BLIP_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: - model = BLIPVisionModelWithProjection.from_pretrained(model_name) - self.assertIsNotNone(model) - self.assertTrue(hasattr(model, "visual_projection")) - -class BLIPTextModelTester: +class BlipTextModelTester: def __init__( self, parent, @@ -266,6 +231,7 @@ def __init__( attention_dropout=0.1, max_position_embeddings=512, initializer_range=0.02, + bos_token_id=0, scope=None, ): self.parent = parent @@ -285,6 +251,7 @@ def __init__( self.max_position_embeddings = max_position_embeddings self.initializer_range = initializer_range self.scope = scope + self.bos_token_id = bos_token_id def prepare_config_and_inputs(self): input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) @@ -305,7 +272,7 @@ def prepare_config_and_inputs(self): return config, input_ids, input_mask def get_config(self): - return BLIPTextConfig( + return BlipTextConfig( vocab_size=self.vocab_size, hidden_size=self.hidden_size, projection_dim=self.projection_dim, @@ -316,10 +283,11 @@ def get_config(self): attention_dropout=self.attention_dropout, max_position_embeddings=self.max_position_embeddings, initializer_range=self.initializer_range, + bos_token_id=self.bos_token_id ) def create_and_check_model(self, config, input_ids, input_mask): - model = BLIPTextModel(config=config) + model = BlipTextModel(config=config) model.to(torch_device) model.eval() with torch.no_grad(): @@ -328,16 +296,6 @@ def create_and_check_model(self, config, input_ids, input_mask): self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size)) self.parent.assertEqual(result.pooler_output.shape, (self.batch_size, self.hidden_size)) - def create_and_check_model_with_projection(self, config, input_ids, input_mask): - model = BLIPTextModelWithProjection(config=config) - model.to(torch_device) - model.eval() - with torch.no_grad(): - result = model(input_ids, attention_mask=input_mask) - result = model(input_ids) - self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size)) - self.parent.assertEqual(result.text_embeds.shape, (self.batch_size, self.projection_dim)) - def prepare_config_and_inputs_for_common(self): config_and_inputs = self.prepare_config_and_inputs() config, input_ids, input_mask = config_and_inputs @@ -346,16 +304,16 @@ def prepare_config_and_inputs_for_common(self): @require_torch -class BLIPTextModelTest(ModelTesterMixin, unittest.TestCase): +class BlipTextModelTest(ModelTesterMixin, unittest.TestCase): - all_model_classes = (BLIPTextModel, BLIPTextModelWithProjection) if is_torch_available() else () + all_model_classes = (BlipTextModel,) if is_torch_available() else () fx_compatible = False test_pruning = False test_head_masking = False def setUp(self): - self.model_tester = BLIPTextModelTester(self) - self.config_tester = ConfigTester(self, config_class=BLIPTextConfig, hidden_size=37) + self.model_tester = BlipTextModelTester(self) + self.config_tester = ConfigTester(self, config_class=BlipTextConfig, hidden_size=37) def test_config(self): self.config_tester.run_common_tests() @@ -364,43 +322,33 @@ def test_model(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_model(*config_and_inputs) - def test_model_with_projection(self): - config_and_inputs = self.model_tester.prepare_config_and_inputs() - self.model_tester.create_and_check_model_with_projection(*config_and_inputs) - def test_training(self): pass def test_training_gradient_checkpointing(self): pass - @unittest.skip(reason="BLIP does not use inputs_embeds") + @unittest.skip(reason="Blip does not use inputs_embeds") def test_inputs_embeds(self): pass - @unittest.skip(reason="BLIPTextModel has no base class and is not available in MODEL_MAPPING") + @unittest.skip(reason="BlipTextModel has no base class and is not available in MODEL_MAPPING") def test_save_load_fast_init_from_base(self): pass - @unittest.skip(reason="BLIPTextModel has no base class and is not available in MODEL_MAPPING") + @unittest.skip(reason="BlipTextModel has no base class and is not available in MODEL_MAPPING") def test_save_load_fast_init_to_base(self): pass @slow def test_model_from_pretrained(self): for model_name in BLIP_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: - model = BLIPTextModel.from_pretrained(model_name) + model = BlipTextModel.from_pretrained(model_name) self.assertIsNotNone(model) - @slow - def test_model_with_projection_from_pretrained(self): - for model_name in BLIP_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: - model = BLIPTextModelWithProjection.from_pretrained(model_name) - self.assertIsNotNone(model) - self.assertTrue(hasattr(model, "text_projection")) -class BLIPModelTester: +class BlipModelTester: def __init__(self, parent, text_kwargs=None, vision_kwargs=None, is_training=True): if text_kwargs is None: @@ -409,8 +357,8 @@ def __init__(self, parent, text_kwargs=None, vision_kwargs=None, is_training=Tru vision_kwargs = {} self.parent = parent - self.text_model_tester = BLIPTextModelTester(parent, **text_kwargs) - self.vision_model_tester = BLIPVisionModelTester(parent, **vision_kwargs) + self.text_model_tester = BlipTextModelTester(parent, **text_kwargs) + self.vision_model_tester = BlipVisionModelTester(parent, **vision_kwargs) self.is_training = is_training def prepare_config_and_inputs(self): @@ -422,12 +370,12 @@ def prepare_config_and_inputs(self): return config, input_ids, attention_mask, pixel_values def get_config(self): - return BLIPConfig.from_text_vision_configs( + return BlipConfig.from_text_vision_configs( self.text_model_tester.get_config(), self.vision_model_tester.get_config(), projection_dim=64 ) def create_and_check_model(self, config, input_ids, attention_mask, pixel_values): - model = BLIPModel(config).to(torch_device).eval() + model = BlipModel(config).to(torch_device).eval() with torch.no_grad(): result = model(input_ids, pixel_values, attention_mask) self.parent.assertEqual( @@ -445,13 +393,14 @@ def prepare_config_and_inputs_for_common(self): "attention_mask": attention_mask, "pixel_values": pixel_values, "return_loss": True, + } return config, inputs_dict @require_torch -class BLIPModelTest(ModelTesterMixin, unittest.TestCase): - all_model_classes = (BLIPModel,) if is_torch_available() else () +class BlipModelTest(ModelTesterMixin, unittest.TestCase): + all_model_classes = (BlipModel,) if is_torch_available() else () fx_compatible = False test_head_masking = False test_pruning = False @@ -459,7 +408,7 @@ class BLIPModelTest(ModelTesterMixin, unittest.TestCase): test_attention_outputs = False def setUp(self): - self.model_tester = BLIPModelTester(self) + self.model_tester = BlipModelTester(self) def test_model(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() @@ -477,11 +426,11 @@ def test_inputs_embeds(self): def test_retain_grad_hidden_states_attentions(self): pass - @unittest.skip(reason="BLIPModel does not have input/output embeddings") + @unittest.skip(reason="BlipModel does not have input/output embeddings") def test_model_common_attributes(self): pass - # override as the `logit_scale` parameter initilization is different for BLIP + # override as the `logit_scale` parameter initilization is different for Blip def test_initialization(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -519,7 +468,7 @@ def _create_and_check_torchscript(self, config, inputs_dict): try: input_ids = inputs_dict["input_ids"] - pixel_values = inputs_dict["pixel_values"] # BLIP needs pixel_values + pixel_values = inputs_dict["pixel_values"] # Blip needs pixel_values traced_model = torch.jit.trace(model, (input_ids, pixel_values)) except RuntimeError: self.fail("Couldn't trace module.") @@ -559,141 +508,235 @@ def _create_and_check_torchscript(self, config, inputs_dict): def test_load_vision_text_config(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - # Save BLIPConfig and check if we can load BLIPVisionConfig from it + # Save BlipConfig and check if we can load BlipVisionConfig from it with tempfile.TemporaryDirectory() as tmp_dir_name: config.save_pretrained(tmp_dir_name) - vision_config = BLIPVisionConfig.from_pretrained(tmp_dir_name) + vision_config = BlipVisionConfig.from_pretrained(tmp_dir_name) self.assertDictEqual(config.vision_config.to_dict(), vision_config.to_dict()) - # Save BLIPConfig and check if we can load BLIPTextConfig from it + # Save BlipConfig and check if we can load BlipTextConfig from it with tempfile.TemporaryDirectory() as tmp_dir_name: config.save_pretrained(tmp_dir_name) - text_config = BLIPTextConfig.from_pretrained(tmp_dir_name) + text_config = BlipTextConfig.from_pretrained(tmp_dir_name) self.assertDictEqual(config.text_config.to_dict(), text_config.to_dict()) - # overwrite from common since FlaxBLIPModel returns nested output - # which is not supported in the common test - @is_pt_flax_cross_test - def test_equivalence_pt_to_flax(self): - config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - for model_class in self.all_model_classes: - with self.subTest(model_class.__name__): + @slow + def test_model_from_pretrained(self): + for model_name in BLIP_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: + model = BlipModel.from_pretrained(model_name) + self.assertIsNotNone(model) + - # load PyTorch class - pt_model = model_class(config).eval() - # Flax models don't use the `use_cache` option and cache is not returned as a default. - # So we disable `use_cache` here for PyTorch model. - pt_model.config.use_cache = False - fx_model_class_name = "Flax" + model_class.__name__ +class BlipTextImageModelsModelTester: + def __init__(self, parent, text_kwargs=None, vision_kwargs=None, is_training=True): + + if text_kwargs is None: + text_kwargs = {} + if vision_kwargs is None: + vision_kwargs = {} + + self.parent = parent + self.text_model_tester = BlipTextModelTester(parent, **text_kwargs) + self.vision_model_tester = BlipVisionModelTester(parent, **vision_kwargs) + self.is_training = is_training + + def prepare_config_and_inputs(self): + text_config, input_ids, attention_mask = self.text_model_tester.prepare_config_and_inputs() + vision_config, pixel_values = self.vision_model_tester.prepare_config_and_inputs() + + config = self.get_config() + + return config, input_ids, attention_mask, pixel_values + + def get_config(self): + return BlipConfig.from_text_vision_configs( + self.text_model_tester.get_config(), self.vision_model_tester.get_config(), projection_dim=64 + ) + + def create_and_check_model(self, config, input_ids, attention_mask, pixel_values): + model = BlipModel(config).to(torch_device).eval() + with torch.no_grad(): + result = model(input_ids, pixel_values, attention_mask) + self.parent.assertEqual( + result.logits_per_image.shape, (self.vision_model_tester.batch_size, self.text_model_tester.batch_size) + ) + self.parent.assertEqual( + result.logits_per_text.shape, (self.text_model_tester.batch_size, self.vision_model_tester.batch_size) + ) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + config, input_ids, attention_mask, pixel_values = config_and_inputs + inputs_dict = { + "input_ids": input_ids, + "attention_mask": attention_mask, + "pixel_values": pixel_values, + + } + return config, inputs_dict + + +@require_torch +class BlipTextImageModelTest(ModelTesterMixin, unittest.TestCase): + all_model_classes = (BlipForConditionalGeneration, BlipForVisualQuestionAnswering, BlipForImageTextRetrieval,) if is_torch_available() else () + fx_compatible = False + test_head_masking = False + test_pruning = False + test_resize_embeddings = False + test_attention_outputs = False - if not hasattr(transformers, fx_model_class_name): - return + def setUp(self): + self.model_tester = BlipTextImageModelsModelTester(self) - fx_model_class = getattr(transformers, fx_model_class_name) + def test_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_model(*config_and_inputs) - # load Flax class - fx_model = fx_model_class(config, dtype=jnp.float32) - # make sure only flax inputs are forward that actually exist in function args - fx_input_keys = inspect.signature(fx_model.__call__).parameters.keys() + @unittest.skip(reason="Hidden_states is tested in individual model tests") + def test_hidden_states_output(self): + pass - # prepare inputs - pt_inputs = self._prepare_for_class(inputs_dict, model_class) + @unittest.skip(reason="Inputs_embeds is tested in individual model tests") + def test_inputs_embeds(self): + pass - # remove function args that don't exist in Flax - pt_inputs = {k: v for k, v in pt_inputs.items() if k in fx_input_keys} + @unittest.skip(reason="Retain_grad is tested in individual model tests") + def test_retain_grad_hidden_states_attentions(self): + pass - fx_state = convert_pytorch_state_dict_to_flax(pt_model.state_dict(), fx_model) - fx_model.params = fx_state + @unittest.skip(reason="BlipModel does not have input/output embeddings") + def test_model_common_attributes(self): + pass - with torch.no_grad(): - pt_outputs = pt_model(**pt_inputs).to_tuple() + def test_training(self): + if not self.model_tester.is_training: + return - # convert inputs to Flax - fx_inputs = {k: np.array(v) for k, v in pt_inputs.items() if torch.is_tensor(v)} - fx_outputs = fx_model(**fx_inputs).to_tuple() - self.assertEqual(len(fx_outputs), len(pt_outputs), "Output lengths differ between Flax and PyTorch") - for fx_output, pt_output in zip(fx_outputs[:4], pt_outputs[:4]): - self.assert_almost_equals(fx_output, pt_output.numpy(), 4e-2) + for model_class in self.all_model_classes[:-1]: + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + config.return_dict = True - with tempfile.TemporaryDirectory() as tmpdirname: - pt_model.save_pretrained(tmpdirname) - fx_model_loaded = fx_model_class.from_pretrained(tmpdirname, from_pt=True) + model = model_class(config) + model.to(torch_device) + model.train() + inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True) + loss = model(**inputs).loss + loss.backward() + + def test_training_gradient_checkpointing(self): + if not self.model_tester.is_training: + return + + for model_class in self.all_model_classes[:-1]: + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + config.use_cache = False + config.return_dict = True - fx_outputs_loaded = fx_model_loaded(**fx_inputs).to_tuple() - self.assertEqual( - len(fx_outputs_loaded), len(pt_outputs), "Output lengths differ between Flax and PyTorch" - ) - for fx_output_loaded, pt_output in zip(fx_outputs_loaded[:4], pt_outputs[:4]): - self.assert_almost_equals(fx_output_loaded, pt_output.numpy(), 4e-2) + model = model_class(config) + model.to(torch_device) + model.gradient_checkpointing_enable() + model.train() + inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True) + loss = model(**inputs).loss + loss.backward() - # overwrite from common since FlaxBLIPModel returns nested output - # which is not supported in the common test - @is_pt_flax_cross_test - def test_equivalence_flax_to_pt(self): + # override as the `logit_scale` parameter initilization is different for Blip + def test_initialization(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + configs_no_init = _config_zero_init(config) for model_class in self.all_model_classes: - with self.subTest(model_class.__name__): - # load corresponding PyTorch class - pt_model = model_class(config).eval() + model = model_class(config=configs_no_init) + for name, param in model.named_parameters(): + if param.requires_grad: + # check if `logit_scale` is initilized as per the original implementation + if name == "logit_scale": + self.assertAlmostEqual( + param.data.item(), + np.log(1 / 0.07), + delta=1e-3, + msg=f"Parameter {name} of model {model_class} seems not properly initialized", + ) + else: + self.assertIn( + ((param.data.mean() * 1e9).round() / 1e9).item(), + [0.0, 1.0], + msg=f"Parameter {name} of model {model_class} seems not properly initialized", + ) - # So we disable `use_cache` here for PyTorch model. - pt_model.config.use_cache = False + def _create_and_check_torchscript(self, config, inputs_dict): + if not self.test_torchscript: + return - fx_model_class_name = "Flax" + model_class.__name__ + configs_no_init = _config_zero_init(config) # To be sure we have no Nan + configs_no_init.torchscript = True + configs_no_init.return_dict = False + for model_class in self.all_model_classes: + model = model_class(config=configs_no_init) + model.to(torch_device) + model.eval() - if not hasattr(transformers, fx_model_class_name): - # no flax model exists for this class - return + try: + input_ids = inputs_dict["input_ids"] + pixel_values = inputs_dict["pixel_values"] # Blip needs pixel_values + traced_model = torch.jit.trace(model, (input_ids, pixel_values)) + except RuntimeError: + self.fail("Couldn't trace module.") - fx_model_class = getattr(transformers, fx_model_class_name) + with tempfile.TemporaryDirectory() as tmp_dir_name: + pt_file_name = os.path.join(tmp_dir_name, "traced_model.pt") - # load Flax class - fx_model = fx_model_class(config, dtype=jnp.float32) - # make sure only flax inputs are forward that actually exist in function args - fx_input_keys = inspect.signature(fx_model.__call__).parameters.keys() + try: + torch.jit.save(traced_model, pt_file_name) + except Exception: + self.fail("Couldn't save module.") - pt_model = load_flax_weights_in_pytorch_model(pt_model, fx_model.params) + try: + loaded_model = torch.jit.load(pt_file_name) + except Exception: + self.fail("Couldn't load module.") - # make sure weights are tied in PyTorch - pt_model.tie_weights() + model.to(torch_device) + model.eval() - # prepare inputs - pt_inputs = self._prepare_for_class(inputs_dict, model_class) + loaded_model.to(torch_device) + loaded_model.eval() - # remove function args that don't exist in Flax - pt_inputs = {k: v for k, v in pt_inputs.items() if k in fx_input_keys} + model_state_dict = model.state_dict() + loaded_model_state_dict = loaded_model.state_dict() - with torch.no_grad(): - pt_outputs = pt_model(**pt_inputs).to_tuple() + self.assertEqual(set(model_state_dict.keys()), set(loaded_model_state_dict.keys())) - fx_inputs = {k: np.array(v) for k, v in pt_inputs.items() if torch.is_tensor(v)} + models_equal = True + for layer_name, p1 in model_state_dict.items(): + p2 = loaded_model_state_dict[layer_name] + if p1.data.ne(p2.data).sum() > 0: + models_equal = False - fx_outputs = fx_model(**fx_inputs).to_tuple() - self.assertEqual(len(fx_outputs), len(pt_outputs), "Output lengths differ between Flax and PyTorch") + self.assertTrue(models_equal) - for fx_output, pt_output in zip(fx_outputs[:4], pt_outputs[:4]): - self.assert_almost_equals(fx_output, pt_output.numpy(), 4e-2) + def test_load_vision_text_config(self): + config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - with tempfile.TemporaryDirectory() as tmpdirname: - fx_model.save_pretrained(tmpdirname) - pt_model_loaded = model_class.from_pretrained(tmpdirname, from_flax=True) + # Save BlipConfig and check if we can load BlipVisionConfig from it + with tempfile.TemporaryDirectory() as tmp_dir_name: + config.save_pretrained(tmp_dir_name) + vision_config = BlipVisionConfig.from_pretrained(tmp_dir_name) + self.assertDictEqual(config.vision_config.to_dict(), vision_config.to_dict()) - with torch.no_grad(): - pt_outputs_loaded = pt_model_loaded(**pt_inputs).to_tuple() + # Save BlipConfig and check if we can load BlipTextConfig from it + with tempfile.TemporaryDirectory() as tmp_dir_name: + config.save_pretrained(tmp_dir_name) + text_config = BlipTextConfig.from_pretrained(tmp_dir_name) + self.assertDictEqual(config.text_config.to_dict(), text_config.to_dict()) - self.assertEqual( - len(fx_outputs), len(pt_outputs_loaded), "Output lengths differ between Flax and PyTorch" - ) - for fx_output, pt_output in zip(fx_outputs[:4], pt_outputs_loaded[:4]): - self.assert_almost_equals(fx_output, pt_output.numpy(), 4e-2) @slow def test_model_from_pretrained(self): for model_name in BLIP_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: - model = BLIPModel.from_pretrained(model_name) + model = BlipModel.from_pretrained(model_name) self.assertIsNotNone(model) @@ -706,11 +749,11 @@ def prepare_img(): @require_vision @require_torch -class BLIPModelIntegrationTest(unittest.TestCase): +class BlipModelIntegrationTest(unittest.TestCase): @slow def test_inference(self): model_name = "ybelkada/blip-base" - model = BLIPModel.from_pretrained(model_name).to(torch_device) + model = BlipModel.from_pretrained(model_name).to(torch_device) processor = CLIPProcessor.from_pretrained(model_name) image = prepare_img() From 5215465271125f97293cd64ba360e3333eda0e1d Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Sun, 11 Dec 2022 12:18:07 +0000 Subject: [PATCH 10/96] fix tests --- src/transformers/__init__.py | 8 +- src/transformers/models/blip/__init__.py | 6 +- .../models/blip/configuration_blip.py | 8 +- src/transformers/models/blip/modeling_blip.py | 94 +++++----- src/transformers/utils/dummy_pt_objects.py | 28 +++ tests/models/blip/test_modeling_blip.py | 36 ++-- tests/models/blip/test_modeling_blip_text.py | 173 ++++++++++++++++++ utils/check_repo.py | 1 + 8 files changed, 281 insertions(+), 73 deletions(-) create mode 100644 tests/models/blip/test_modeling_blip_text.py diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 597fb193462e..0ac684cb99ec 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -1096,12 +1096,12 @@ [ "BLIP_PRETRAINED_MODEL_ARCHIVE_LIST", "BlipForConditionalGeneration", + "BlipForImageTextRetrieval", + "BlipForVisualQuestionAnswering", "BlipModel", "BlipPreTrainedModel", - "BlipForVisualQuestionAnswering", "BlipTextModel", "BlipVisionModel", - "BlipForImageTextRetrieval", ] ) _import_structure["models.bloom"].extend( @@ -4252,12 +4252,12 @@ from .models.blip import ( BLIP_PRETRAINED_MODEL_ARCHIVE_LIST, BlipForConditionalGeneration, + BlipForImageTextRetrieval, BlipForVisualQuestionAnswering, BlipModel, + BlipPreTrainedModel, BlipTextModel, BlipVisionModel, - BlipPreTrainedModel, - BlipForImageTextRetrieval, ) from .models.bloom import ( BLOOM_PRETRAINED_MODEL_ARCHIVE_LIST, diff --git a/src/transformers/models/blip/__init__.py b/src/transformers/models/blip/__init__.py index 8f28687f205a..1938c571d4cc 100644 --- a/src/transformers/models/blip/__init__.py +++ b/src/transformers/models/blip/__init__.py @@ -58,12 +58,12 @@ from .modeling_blip import ( BLIP_PRETRAINED_MODEL_ARCHIVE_LIST, BlipForConditionalGeneration, + BlipForImageTextRetrieval, BlipForVisualQuestionAnswering, BlipModel, - BlipVisionModel, - BlipTextModel, - BlipForImageTextRetrieval, BlipPreTrainedModel, + BlipTextModel, + BlipVisionModel, ) else: diff --git a/src/transformers/models/blip/configuration_blip.py b/src/transformers/models/blip/configuration_blip.py index 32464107176c..637073e87b1b 100644 --- a/src/transformers/models/blip/configuration_blip.py +++ b/src/transformers/models/blip/configuration_blip.py @@ -313,7 +313,13 @@ class BlipConfig(PretrainedConfig): is_composition = True def __init__( - self, text_config=None, vision_config=None, projection_dim=512, logit_scale_init_value=2.6592, image_text_hidden_size=256,**kwargs + self, + text_config=None, + vision_config=None, + projection_dim=512, + logit_scale_init_value=2.6592, + image_text_hidden_size=256, + **kwargs ): super().__init__(**kwargs) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 0bfad9436491..3df85c73489c 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -17,12 +17,12 @@ from typing import Any, Optional, Tuple, Union import torch +import torch.nn.functional as F import torch.utils.checkpoint from torch import nn -import torch.nn.functional as F from ...activations import ACT2FN -from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling, CausalLMOutputWithCrossAttentions +from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling from ...modeling_utils import PreTrainedModel from ...utils import ( ModelOutput, @@ -43,6 +43,8 @@ "ybelkada/blip-base", # See all BLIP models at https://huggingface.co/models?filter=blip ] + + # contrastive loss function, adapted from # https://sachinruk.github.io/blog/pytorch/pytorch%20lightning/loss%20function/gpu/2021/03/07/BLIP.html def contrastive_loss(logits: torch.Tensor) -> torch.Tensor: @@ -55,8 +57,8 @@ def blip_loss(similarity: torch.Tensor) -> torch.Tensor: image_loss = contrastive_loss(similarity.t()) return (caption_loss + image_loss) / 2.0 + @dataclass -# Copied from transformers.models.clip.modeling_clip.CLIPVisionModelOutput with CLIP->BLIP class BlipForConditionalGenerationModelOutput(ModelOutput): """ Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states. @@ -78,6 +80,7 @@ class BlipForConditionalGenerationModelOutput(ModelOutput): Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. """ + loss: Optional[Tuple[torch.FloatTensor]] = None decoder_logits: Optional[Tuple[torch.FloatTensor]] = None image_embeds: Optional[torch.FloatTensor] = None @@ -87,7 +90,6 @@ class BlipForConditionalGenerationModelOutput(ModelOutput): @dataclass -# Copied from transformers.models.clip.modeling_clip.CLIPVisionModelOutput with CLIP->BLIP class BlipVisionModelOutput(ModelOutput): """ Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states. @@ -109,6 +111,7 @@ class BlipVisionModelOutput(ModelOutput): Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. """ + loss: Optional[torch.FloatTensor] = None image_embeds: Optional[torch.FloatTensor] = None last_hidden_state: torch.FloatTensor = None @@ -211,7 +214,6 @@ def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: return embeddings - class BlipTextEmbeddings(nn.Module): def __init__(self, config: BlipTextConfig): super().__init__() @@ -721,7 +723,7 @@ def forward( hidden_states=encoder_outputs.hidden_states, attentions=encoder_outputs.attentions, ) - + def get_input_embeddings(self): return self.embeddings @@ -1110,7 +1112,6 @@ class BlipForVisualQuestionAnswering(BlipPreTrainedModel): config_class = BlipConfig _keys_to_ignore_on_load_missing = [r"text_decoder.cls.predictions.decoder.bias"] - def __init__(self, config: BlipConfig): super().__init__(config) @@ -1177,28 +1178,28 @@ def forward( image_atts = torch.ones(image_embeds.size()[:-1], dtype=torch.long) question_embeds = self.text_encoder( - input_ids=input_ids, - attention_mask=attention_mask, + input_ids=input_ids, + attention_mask=attention_mask, encoder_hidden_states=image_embeds, - encoder_attention_mask=image_atts, - return_dict=return_dict, + encoder_attention_mask=image_atts, + return_dict=return_dict, ) question_embeds = question_embeds[0] if not return_dict else question_embeds.last_hidden_state - + if answer_input_ids is None: answer_input_ids = torch.LongTensor([self.decoder_bos_token_id]).repeat((batch_size, 1)) - answer_targets = answer_input_ids.masked_fill(answer_input_ids == self.decoder_pad_token_id, -100) + answer_targets = answer_input_ids.masked_fill(answer_input_ids == self.decoder_pad_token_id, -100) answer_output = self.text_decoder( - input_ids=answer_input_ids, - attention_mask=answer_attention_mask, + input_ids=answer_input_ids, + attention_mask=answer_attention_mask, encoder_hidden_states=question_embeds, - encoder_attention_mask=attention_mask, + encoder_attention_mask=attention_mask, labels=answer_targets, - return_dict=return_dict, - reduction='none', - ) + return_dict=return_dict, + reduction="none", + ) decoder_loss = answer_output.loss.mean() if return_dict else answer_output[0].mean() @@ -1213,14 +1214,14 @@ def forward( hidden_states=vision_outputs.hidden_states, attentions=vision_outputs.attentions, ) - + @torch.no_grad() def generate( - self, - input_ids: torch.LongTensor, - pixel_values: torch.FloatTensor, - attention_mask: Optional[torch.LongTensor]=None, - use_rank_mode: Optional[bool]=False, + self, + input_ids: torch.LongTensor, + pixel_values: torch.FloatTensor, + attention_mask: Optional[torch.LongTensor] = None, + use_rank_mode: Optional[bool] = False, **generate_kwargs ) -> torch.LongTensor: r""" @@ -1264,19 +1265,21 @@ def generate( input_ids = torch.LongTensor(input_ids) question_outputs = self.text_encoder( - input_ids=input_ids, - attention_mask=attention_mask, + input_ids=input_ids, + attention_mask=attention_mask, encoder_hidden_states=image_embeds, encoder_attention_mask=image_atts, - return_dict=False + return_dict=False, ) question_embeds = question_outputs[0] - question_atts = torch.ones(question_embeds.size()[:-1],dtype=torch.long).to(question_embeds.device) - model_kwargs = {"encoder_hidden_states": question_embeds, "encoder_attention_mask":question_atts} + question_atts = torch.ones(question_embeds.size()[:-1], dtype=torch.long).to(question_embeds.device) + model_kwargs = {"encoder_hidden_states": question_embeds, "encoder_attention_mask": question_atts} - bos_ids = torch.full((question_embeds.size(0),1),fill_value=self.decoder_bos_token_id,device=question_embeds.device) + bos_ids = torch.full( + (question_embeds.size(0), 1), fill_value=self.decoder_bos_token_id, device=question_embeds.device + ) outputs = self.text_decoder.generate( input_ids=bos_ids, @@ -1288,6 +1291,7 @@ def generate( return outputs + @add_start_docstrings( """ BLIP Vision Model with a projection layer on top (a linear layer on top of the pooled output). @@ -1307,7 +1311,7 @@ def __init__(self, config: BlipConfig): self.vision_proj = nn.Linear(config.vision_config.hidden_size, config.image_text_hidden_size) self.text_proj = nn.Linear(config.text_config.hidden_size, config.image_text_hidden_size) - self.itm_head = nn.Linear(config.text_config.hidden_size, 2) + self.itm_head = nn.Linear(config.text_config.hidden_size, 2) self.decoder_pad_token_id = config.text_config.pad_token_id self.decoder_bos_token_id = config.text_config.bos_token_id @@ -1366,33 +1370,31 @@ def forward( if use_itm_head: question_embeds = self.text_encoder( - input_ids=input_ids, - attention_mask=attention_mask, + input_ids=input_ids, + attention_mask=attention_mask, encoder_hidden_states=image_embeds, - encoder_attention_mask=image_atts, - return_dict=return_dict, + encoder_attention_mask=image_atts, + return_dict=return_dict, ) question_embeds = question_embeds[0] if not return_dict else question_embeds.last_hidden_state - output = self.itm_head(question_embeds[:,0,:]) + output = self.itm_head(question_embeds[:, 0, :]) else: question_embeds = self.text_encoder( - input_ids=input_ids, - attention_mask=attention_mask, + input_ids=input_ids, + attention_mask=attention_mask, return_dict=return_dict, mode="text", ) question_embeds = question_embeds[0] if not return_dict else question_embeds.last_hidden_state - image_feat = F.normalize(self.vision_proj(image_embeds[:,0,:]),dim=-1) - text_feat = F.normalize(self.text_proj(question_embeds[:,0,:]),dim=-1) - - output = image_feat @ text_feat.t() - + image_feat = F.normalize(self.vision_proj(image_embeds[:, 0, :]), dim=-1) + text_feat = F.normalize(self.text_proj(question_embeds[:, 0, :]), dim=-1) + output = image_feat @ text_feat.t() if not return_dict: - outputs = (output, vision_outputs[0]) + vision_outputs[2:] + outputs = (output, vision_outputs[0]) + vision_outputs[2:] return tuple(output for output in outputs if output is not None) return BlipVisionModelOutput( @@ -1400,4 +1402,4 @@ def forward( last_hidden_state=vision_outputs.last_hidden_state, hidden_states=vision_outputs.hidden_states, attentions=vision_outputs.attentions, - ) \ No newline at end of file + ) diff --git a/src/transformers/utils/dummy_pt_objects.py b/src/transformers/utils/dummy_pt_objects.py index 6e9616e3688a..cd1c612bb889 100644 --- a/src/transformers/utils/dummy_pt_objects.py +++ b/src/transformers/utils/dummy_pt_objects.py @@ -1122,6 +1122,20 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["torch"]) +class BlipForImageTextRetrieval(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class BlipForVisualQuestionAnswering(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + class BlipModel(metaclass=DummyObject): _backends = ["torch"] @@ -1136,6 +1150,20 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["torch"]) +class BlipTextModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + +class BlipVisionModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + BLOOM_PRETRAINED_MODEL_ARCHIVE_LIST = None diff --git a/tests/models/blip/test_modeling_blip.py b/tests/models/blip/test_modeling_blip.py index 2807887bcfdb..dbb3530a0fd5 100644 --- a/tests/models/blip/test_modeling_blip.py +++ b/tests/models/blip/test_modeling_blip.py @@ -24,12 +24,7 @@ import requests from transformers import BlipConfig, BlipTextConfig, BlipVisionConfig -from transformers.testing_utils import ( - require_torch, - require_vision, - slow, - torch_device, -) +from transformers.testing_utils import require_torch, require_vision, slow, torch_device from transformers.utils import is_torch_available, is_vision_available from ...test_configuration_common import ConfigTester @@ -47,12 +42,12 @@ from torch import nn from transformers import ( + BlipForConditionalGeneration, + BlipForImageTextRetrieval, + BlipForVisualQuestionAnswering, BlipModel, BlipTextModel, BlipVisionModel, - BlipForConditionalGeneration, - BlipForVisualQuestionAnswering, - BlipForImageTextRetrieval, ) from transformers.models.blip.modeling_blip import BLIP_PRETRAINED_MODEL_ARCHIVE_LIST @@ -62,6 +57,7 @@ from transformers import CLIPProcessor + class BlipVisionModelTester: def __init__( self, @@ -283,7 +279,7 @@ def get_config(self): attention_dropout=self.attention_dropout, max_position_embeddings=self.max_position_embeddings, initializer_range=self.initializer_range, - bos_token_id=self.bos_token_id + bos_token_id=self.bos_token_id, ) def create_and_check_model(self, config, input_ids, input_mask): @@ -347,7 +343,6 @@ def test_model_from_pretrained(self): self.assertIsNotNone(model) - class BlipModelTester: def __init__(self, parent, text_kwargs=None, vision_kwargs=None, is_training=True): @@ -393,7 +388,6 @@ def prepare_config_and_inputs_for_common(self): "attention_mask": attention_mask, "pixel_values": pixel_values, "return_loss": True, - } return config, inputs_dict @@ -520,7 +514,6 @@ def test_load_vision_text_config(self): text_config = BlipTextConfig.from_pretrained(tmp_dir_name) self.assertDictEqual(config.text_config.to_dict(), text_config.to_dict()) - @slow def test_model_from_pretrained(self): for model_name in BLIP_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: @@ -528,7 +521,6 @@ def test_model_from_pretrained(self): self.assertIsNotNone(model) - class BlipTextImageModelsModelTester: def __init__(self, parent, text_kwargs=None, vision_kwargs=None, is_training=True): @@ -573,14 +565,21 @@ def prepare_config_and_inputs_for_common(self): "input_ids": input_ids, "attention_mask": attention_mask, "pixel_values": pixel_values, - } return config, inputs_dict @require_torch class BlipTextImageModelTest(ModelTesterMixin, unittest.TestCase): - all_model_classes = (BlipForConditionalGeneration, BlipForVisualQuestionAnswering, BlipForImageTextRetrieval,) if is_torch_available() else () + all_model_classes = ( + ( + BlipForConditionalGeneration, + BlipForVisualQuestionAnswering, + BlipForImageTextRetrieval, + ) + if is_torch_available() + else () + ) fx_compatible = False test_head_masking = False test_pruning = False @@ -624,11 +623,11 @@ def test_training(self): inputs = self._prepare_for_class(inputs_dict, model_class, return_labels=True) loss = model(**inputs).loss loss.backward() - + def test_training_gradient_checkpointing(self): if not self.model_tester.is_training: return - + for model_class in self.all_model_classes[:-1]: config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() config.use_cache = False @@ -732,7 +731,6 @@ def test_load_vision_text_config(self): text_config = BlipTextConfig.from_pretrained(tmp_dir_name) self.assertDictEqual(config.text_config.to_dict(), text_config.to_dict()) - @slow def test_model_from_pretrained(self): for model_name in BLIP_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: diff --git a/tests/models/blip/test_modeling_blip_text.py b/tests/models/blip/test_modeling_blip_text.py new file mode 100644 index 000000000000..c1fb06354c0b --- /dev/null +++ b/tests/models/blip/test_modeling_blip_text.py @@ -0,0 +1,173 @@ +# coding=utf-8 +# Copyright 2022 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Testing suite for the PyTorch Blip model. """ +import unittest + +import numpy as np + +from transformers import BlipTextConfig +from transformers.testing_utils import require_torch, slow, torch_device +from transformers.utils import is_torch_available + +from ...test_configuration_common import ConfigTester +from ...test_modeling_common import ( + ModelTesterMixin, + ids_tensor, + random_attention_mask, +) + + +if is_torch_available(): + import torch + + from transformers import ( + BlipTextModel, + ) + from transformers.models.blip.modeling_blip import BLIP_PRETRAINED_MODEL_ARCHIVE_LIST + + +class BlipTextModelTester: + def __init__( + self, + parent, + batch_size=12, + seq_length=7, + is_training=True, + use_input_mask=True, + use_labels=True, + vocab_size=99, + hidden_size=32, + projection_dim=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + dropout=0.1, + attention_dropout=0.1, + max_position_embeddings=512, + initializer_range=0.02, + bos_token_id=0, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_mask = use_input_mask + self.use_labels = use_labels + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.projection_dim = projection_dim + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.dropout = dropout + self.attention_dropout = attention_dropout + self.max_position_embeddings = max_position_embeddings + self.initializer_range = initializer_range + self.scope = scope + self.bos_token_id = bos_token_id + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = random_attention_mask([self.batch_size, self.seq_length]) + + if input_mask is not None: + batch_size, seq_length = input_mask.shape + rnd_start_indices = np.random.randint(1, seq_length - 1, size=(batch_size,)) + for batch_idx, start_index in enumerate(rnd_start_indices): + input_mask[batch_idx, :start_index] = 1 + input_mask[batch_idx, start_index:] = 0 + + config = self.get_config() + + return config, input_ids, input_mask + + def get_config(self): + return BlipTextConfig( + vocab_size=self.vocab_size, + hidden_size=self.hidden_size, + projection_dim=self.projection_dim, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + intermediate_size=self.intermediate_size, + dropout=self.dropout, + attention_dropout=self.attention_dropout, + max_position_embeddings=self.max_position_embeddings, + initializer_range=self.initializer_range, + bos_token_id=self.bos_token_id, + ) + + def create_and_check_model(self, config, input_ids, input_mask): + model = BlipTextModel(config=config) + model.to(torch_device) + model.eval() + with torch.no_grad(): + result = model(input_ids, attention_mask=input_mask) + result = model(input_ids) + self.parent.assertEqual(result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size)) + self.parent.assertEqual(result.pooler_output.shape, (self.batch_size, self.hidden_size)) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + config, input_ids, input_mask = config_and_inputs + inputs_dict = {"input_ids": input_ids, "attention_mask": input_mask} + return config, inputs_dict + + +@require_torch +class BlipTextModelTest(ModelTesterMixin, unittest.TestCase): + + all_model_classes = (BlipTextModel,) if is_torch_available() else () + fx_compatible = False + test_pruning = False + test_head_masking = False + + def setUp(self): + self.model_tester = BlipTextModelTester(self) + self.config_tester = ConfigTester(self, config_class=BlipTextConfig, hidden_size=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_model(*config_and_inputs) + + def test_training(self): + pass + + def test_training_gradient_checkpointing(self): + pass + + @unittest.skip(reason="Blip does not use inputs_embeds") + def test_inputs_embeds(self): + pass + + @unittest.skip(reason="BlipTextModel has no base class and is not available in MODEL_MAPPING") + def test_save_load_fast_init_from_base(self): + pass + + @unittest.skip(reason="BlipTextModel has no base class and is not available in MODEL_MAPPING") + def test_save_load_fast_init_to_base(self): + pass + + @slow + def test_model_from_pretrained(self): + for model_name in BLIP_PRETRAINED_MODEL_ARCHIVE_LIST[:1]: + model = BlipTextModel.from_pretrained(model_name) + self.assertIsNotNone(model) diff --git a/utils/check_repo.py b/utils/check_repo.py index 0f61fafd63da..7d0b3294bbdc 100644 --- a/utils/check_repo.py +++ b/utils/check_repo.py @@ -120,6 +120,7 @@ "FlaxBertForCausalLM", # Building part of bigger (tested) model. Tested implicitly through FlaxRobertaForCausalLM. "OPTDecoderWrapper", "TFSegformerDecodeHead", # Not a regular model. + "BlipTextLMHeadModel", # No need to test it as it is tested by BlipTextVision models ] # Update this list with test files that don't have a tester with a `all_model_classes` variable and which don't From da07c0f88cbc8f6684bba1ee281a2d62a07f4f70 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Sun, 11 Dec 2022 12:26:17 +0000 Subject: [PATCH 11/96] make fixup --- docs/source/en/model_doc/blip.mdx | 39 +++++---- .../models/blip/modeling_blip_text.py | 86 +++++++++---------- tests/models/blip/test_modeling_blip_text.py | 10 +-- utils/check_repo.py | 6 ++ 4 files changed, 72 insertions(+), 69 deletions(-) diff --git a/docs/source/en/model_doc/blip.mdx b/docs/source/en/model_doc/blip.mdx index 468f80b07e9f..83144545b6eb 100644 --- a/docs/source/en/model_doc/blip.mdx +++ b/docs/source/en/model_doc/blip.mdx @@ -29,43 +29,50 @@ This model was contributed by [INSERT YOUR HF USERNAME HERE](https://huggingface The original code can be found [here](). -## BLIPConfig +## BlipConfig -[[autodoc]] BLIPConfig +[[autodoc]] BlipConfig - from_text_vision_configs -## BLIPTextConfig +## BlipTextConfig -[[autodoc]] BLIPTextConfig +[[autodoc]] BlipTextConfig -## BLIPVisionConfig +## BlipVisionConfig -[[autodoc]] BLIPVisionConfig +[[autodoc]] BlipVisionConfig -## BLIPModel +## BlipModel -[[autodoc]] BLIPModel +[[autodoc]] BlipModel - forward - get_text_features - get_image_features -## BLIPTextModel +## BlipTextModel -[[autodoc]] BLIPTextModel +[[autodoc]] BlipTextModel - forward -## BLIPTextModelWithProjection -[[autodoc]] BLIPTextModelWithProjection +## BlipVisionModel + +[[autodoc]] BlipVisionModel - forward -## BLIPVisionModelWithProjection -[[autodoc]] BLIPVisionModelWithProjection +## BlipForConditionalGeneration + +[[autodoc]] BlipForConditionalGeneration - forward -## BLIPVisionModel +## BlipForImageTextRetrieval + +[[autodoc]] BlipForImageTextRetrieval + - forward + +## BlipForVisualQuestionAnswering -[[autodoc]] BLIPVisionModel +[[autodoc]] BlipForVisualQuestionAnswering - forward diff --git a/src/transformers/models/blip/modeling_blip_text.py b/src/transformers/models/blip/modeling_blip_text.py index 7dd7c9805ea2..6660090fd34d 100644 --- a/src/transformers/models/blip/modeling_blip_text.py +++ b/src/transformers/models/blip/modeling_blip_text.py @@ -555,10 +555,10 @@ def _init_weights(self, module): class BlipTextModel(BlipTextPreTrainedModel): """ The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of - cross-attention is added between the self-attention layers, following the architecture described in `Attention is - all you need `__ by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, - Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. argument and :obj:`add_cross_attention` set to - :obj:`True`; an :obj:`encoder_hidden_states` is then expected as an input to the forward pass. + cross-attention is added between the self-attention layers, following the architecture described in [Attention is + all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, + Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. argument and `add_cross_attention` set to `True`; + an `encoder_hidden_states` is then expected as an input to the forward pass. """ def __init__(self, config, add_pooling_layer=True): @@ -594,15 +594,15 @@ def get_extended_attention_mask( Makes broadcastable attention and causal masks so that future and masked tokens are ignored. Arguments: - attention_mask (:obj:`torch.Tensor`): + attention_mask (`torch.Tensor`): Mask with ones indicating tokens to attend to, zeros for tokens to ignore. - input_shape (:obj:`Tuple[int]`): + input_shape (`Tuple[int]`): The shape of the input to the model. - device: (:obj:`torch.device`): + device: (`torch.device`): The device of the input to the model. Returns: - :obj:`torch.Tensor` The extended attention mask, with a the same dtype as :obj:`attention_mask.dtype`. + `torch.Tensor` The extended attention mask, with a the same dtype as `attention_mask.dtype`. """ # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] # ourselves in which case we just need to make it broadcastable to all heads. @@ -672,24 +672,24 @@ def forward( ): r""" encoder_hidden_states (: - obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`): Sequence - of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model - is configured as a decoder. - encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + obj:*torch.FloatTensor* of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Sequence of + hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model is + configured as a decoder. + encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``: + the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. past_key_values (: - obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of - shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): Contains precomputed key - and value hidden states of the attention blocks. Can be used to speed up decoding. If - :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids` - (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)` - instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`. - use_cache (:obj:`bool`, `optional`): - If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up - decoding (see :obj:`past_key_values`). + obj:*tuple(tuple(torch.FloatTensor))* of length `config.n_layers` with each tuple having 4 tensors of shape + `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): Contains precomputed key and value + hidden states of the attention blocks. Can be used to speed up decoding. If `past_key_values` are used, the + user can optionally input only the last `decoder_input_ids` (those that don't have their past key value + states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape + `(batch_size, sequence_length)`. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). """ output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions output_hidden_states = ( @@ -835,35 +835,31 @@ def forward( ): r""" encoder_hidden_states (: - obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length, hidden_size)`, `optional`): Sequence - of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model - is configured as a decoder. - encoder_attention_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + obj:*torch.FloatTensor* of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Sequence of + hidden-states at the output of the last layer of the encoder. Used in the cross-attention if the model is + configured as a decoder. + encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in - the cross-attention if the model is configured as a decoder. Mask values selected in ``[0, 1]``: + the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. - labels (:obj:`torch.LongTensor` of shape :obj:`(batch_size, sequence_length)`, `optional`): + labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in - ``[-100, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) Tokens with indices set to ``-100`` are - ignored (masked), the loss is only computed for the tokens with labels n ``[0, ..., config.vocab_size]`` + `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are + ignored (masked), the loss is only computed for the tokens with labels n `[0, ..., config.vocab_size]` past_key_values (: - obj:`tuple(tuple(torch.FloatTensor))` of length :obj:`config.n_layers` with each tuple having 4 tensors of - shape :obj:`(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): Contains precomputed key - and value hidden states of the attention blocks. Can be used to speed up decoding. If - :obj:`past_key_values` are used, the user can optionally input only the last :obj:`decoder_input_ids` - (those that don't have their past key value states given to this model) of shape :obj:`(batch_size, 1)` - instead of all :obj:`decoder_input_ids` of shape :obj:`(batch_size, sequence_length)`. - use_cache (:obj:`bool`, `optional`): - If set to :obj:`True`, :obj:`past_key_values` key value states are returned and can be used to speed up - decoding (see :obj:`past_key_values`). + obj:*tuple(tuple(torch.FloatTensor))* of length `config.n_layers` with each tuple having 4 tensors of shape + `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): Contains precomputed key and value + hidden states of the attention blocks. Can be used to speed up decoding. If `past_key_values` are used, the + user can optionally input only the last `decoder_input_ids` (those that don't have their past key value + states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape + `(batch_size, sequence_length)`. + use_cache (`bool`, *optional*): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). Returns: - Example:: - >>> from transformers import BlipTextTokenizer, BlipTextLMHeadModel, BlipTextConfig >>> import torch >>> - tokenizer = BlipTextTokenizer.from_pretrained('bert-base-cased') >>> config = - BlipTextConfig.from_pretrained("bert-base-cased") >>> model = - BlipTextLMHeadModel.from_pretrained('bert-base-cased', config=config) >>> inputs = tokenizer("Hello, my dog - is cute", return_tensors="pt") >>> outputs = model(**inputs) >>> prediction_logits = outputs.logits + Example: + """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict if labels is not None: diff --git a/tests/models/blip/test_modeling_blip_text.py b/tests/models/blip/test_modeling_blip_text.py index c1fb06354c0b..2e5e37ce2e96 100644 --- a/tests/models/blip/test_modeling_blip_text.py +++ b/tests/models/blip/test_modeling_blip_text.py @@ -22,19 +22,13 @@ from transformers.utils import is_torch_available from ...test_configuration_common import ConfigTester -from ...test_modeling_common import ( - ModelTesterMixin, - ids_tensor, - random_attention_mask, -) +from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask if is_torch_available(): import torch - from transformers import ( - BlipTextModel, - ) + from transformers import BlipTextModel from transformers.models.blip.modeling_blip import BLIP_PRETRAINED_MODEL_ARCHIVE_LIST diff --git a/utils/check_repo.py b/utils/check_repo.py index 7d0b3294bbdc..0f61b5fbf719 100644 --- a/utils/check_repo.py +++ b/utils/check_repo.py @@ -147,6 +147,12 @@ # should **not** be the rule. IGNORE_NON_AUTO_CONFIGURED = PRIVATE_MODELS.copy() + [ # models to ignore for model xxx mapping + "BlipForConditionalGeneration", + "BlipForImageTextRetrieval", + "BlipForVisualQuestionAnswering", + "BlipVisionModel", + "BlipTextLMHeadModel", + "BlipTextModel", "CLIPSegForImageSegmentation", "CLIPSegVisionModel", "CLIPSegTextModel", From 859158a5eb43cf38d75639e8c2cdf65e2f34a26d Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Sun, 11 Dec 2022 12:33:01 +0000 Subject: [PATCH 12/96] add to `toctree` --- docs/source/en/_toctree.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/en/_toctree.yml b/docs/source/en/_toctree.yml index 57217885cae8..e0ff20494c94 100644 --- a/docs/source/en/_toctree.yml +++ b/docs/source/en/_toctree.yml @@ -492,6 +492,8 @@ title: Audio models - isExpanded: false sections: + - local: model_doc/blip + title: BLIP - local: model_doc/chinese_clip title: Chinese-CLIP - local: model_doc/clip From ee56ba0745ceea67ab03de21a3673d633936b074 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Sun, 11 Dec 2022 12:49:44 +0000 Subject: [PATCH 13/96] fix processors --- .../models/auto/feature_extraction_auto.py | 1 - .../models/auto/image_processing_auto.py | 2 +- src/transformers/models/blip/__init__.py | 21 +- .../models/blip/feature_extraction_blip.py | 490 ++++++++++++++++++ .../models/blip/processing_blip.py | 114 ++++ 5 files changed, 625 insertions(+), 3 deletions(-) create mode 100644 src/transformers/models/blip/feature_extraction_blip.py create mode 100644 src/transformers/models/blip/processing_blip.py diff --git a/src/transformers/models/auto/feature_extraction_auto.py b/src/transformers/models/auto/feature_extraction_auto.py index 4aeb0855edf8..a33affe3ec9b 100644 --- a/src/transformers/models/auto/feature_extraction_auto.py +++ b/src/transformers/models/auto/feature_extraction_auto.py @@ -39,7 +39,6 @@ [ ("audio-spectrogram-transformer", "ASTFeatureExtractor"), ("beit", "BeitFeatureExtractor"), - ("blip", "BLIPFeatureExtractor"), ("chinese_clip", "ChineseCLIPFeatureExtractor"), ("clip", "CLIPFeatureExtractor"), ("clipseg", "ViTFeatureExtractor"), diff --git a/src/transformers/models/auto/image_processing_auto.py b/src/transformers/models/auto/image_processing_auto.py index 0fe4cb72972b..9cf245e2ee31 100644 --- a/src/transformers/models/auto/image_processing_auto.py +++ b/src/transformers/models/auto/image_processing_auto.py @@ -39,7 +39,7 @@ [ ("beit", "BeitImageProcessor"), ("bit", "BitImageProcessor"), - ("blip", "BLIPImageProcessor"), + ("blip", "BlipImageProcessor"), ("chinese_clip", "ChineseCLIPImageProcessor"), ("clip", "CLIPImageProcessor"), ("clipseg", "ViTImageProcessor"), diff --git a/src/transformers/models/blip/__init__.py b/src/transformers/models/blip/__init__.py index 1938c571d4cc..39e8e124a090 100644 --- a/src/transformers/models/blip/__init__.py +++ b/src/transformers/models/blip/__init__.py @@ -17,7 +17,7 @@ # limitations under the License. from typing import TYPE_CHECKING -from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available +from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available _import_structure = { @@ -29,6 +29,16 @@ ], } +try: + if not is_vision_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + pass +else: + _import_structure["feature_extraction_vilt"] = ["BlipFeatureExtractor"] + _import_structure["processing_vilt"] = ["BlipProcessor"] + + try: if not is_torch_available(): raise OptionalDependencyNotAvailable() @@ -49,6 +59,15 @@ if TYPE_CHECKING: from .configuration_blip import BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP, BlipConfig, BlipTextConfig, BlipVisionConfig + try: + if not is_vision_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + pass + else: + from .feature_extraction_blip import BlipFeatureExtractor + from .processing_blip import BlipProcessor + try: if not is_torch_available(): raise OptionalDependencyNotAvailable() diff --git a/src/transformers/models/blip/feature_extraction_blip.py b/src/transformers/models/blip/feature_extraction_blip.py new file mode 100644 index 000000000000..52ba3ee49e33 --- /dev/null +++ b/src/transformers/models/blip/feature_extraction_blip.py @@ -0,0 +1,490 @@ +# coding=utf-8 +# Copyright 2022 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Feature extractor class for ViLT.""" + +from ...utils import logging + +import warnings +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union + +import numpy as np + +from transformers.utils import is_vision_available +from transformers.utils.generic import TensorType + +from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict +from ...image_transforms import normalize, rescale, resize, to_channel_dimension_format +from ...image_utils import ( + IMAGENET_STANDARD_MEAN, + IMAGENET_STANDARD_STD, + ChannelDimension, + ImageInput, + PILImageResampling, + get_image_size, + infer_channel_dimension_format, + is_batched, + to_numpy_array, + valid_images, +) +from ...utils import logging + + +if is_vision_available(): + import PIL + + +logger = logging.get_logger(__name__) + + +def max_across_indices(values: Iterable[Any]) -> List[Any]: + """ + Return the maximum value across all indices of an iterable of values. + """ + return [max(values_i) for values_i in zip(*values)] + + +def pad( + image: np.ndarray, + output_size: Tuple[int, int], + input_channel_dimension: Optional[ChannelDimension] = None, + data_format: Optional[ChannelDimension] = None, +) -> np.ndarray: + """ + Pad the bottom and right of the image with zeros to the output size. + + Args: + image (`np.ndarray`): + Image to pad. + output_size (`Tuple[int, int]`): + Output size of the image. + input_channel_dimension (`ChannelDimension`, *optional*): + The channel dimension format of the image. If not provided, it will be inferred from the input image. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format of the image. If not provided, it will be the same as the input image. + """ + if input_channel_dimension is None: + input_channel_dimension = infer_channel_dimension_format(image) + + output_height, output_width = output_size + input_height, input_width = get_image_size(image) + pad_bottom = output_height - input_height + pad_right = output_width - input_width + + if input_channel_dimension == ChannelDimension.FIRST: + padded_image = np.pad(image, [(0, 0), (0, pad_bottom), (0, pad_right)], mode="constant", constant_values=0) + elif input_channel_dimension == ChannelDimension.LAST: + padded_image = np.pad(image, [(0, pad_bottom), (0, pad_right), (0, 0)], mode="constant", constant_values=0) + else: + raise ValueError(f"Invalid channel dimension format: {input_channel_dimension}") + + if data_format is not None: + padded_image = to_channel_dimension_format(padded_image, data_format) + + return padded_image + + +def make_pixel_mask(image: np.ndarray, output_size: Tuple[int, int]) -> np.ndarray: + """ + Make a pixel mask for the image, where 1 indicates a valid pixel and 0 indicates padding. + + Args: + image (`np.ndarray`): + Image to make the pixel mask for. + output_size (`Tuple[int, int]`): + Output size of the mask. + """ + input_height, input_width = get_image_size(image) + mask = np.zeros(output_size, dtype=np.int64) + mask[:input_height, :input_width] = 1 + return mask + + +def get_max_dimensions(images: List[np.ndarray]) -> List[int]: + """ + Get the maximum height and width across all images in a batch. + """ + input_channel_dimension = infer_channel_dimension_format(images[0]) + + if input_channel_dimension == ChannelDimension.FIRST: + _, max_height, max_width = max_across_indices([img.shape for img in images]) + elif input_channel_dimension == ChannelDimension.LAST: + max_height, max_width, _ = max_across_indices([img.shape for img in images]) + else: + raise ValueError(f"Invalid channel dimension format: {input_channel_dimension}") + return (max_height, max_width) + + +def get_resize_output_image_size( + input_image: np.ndarray, shorter: int = 800, longer: int = 1333, size_divisor: int = 32 +) -> Tuple[int, int]: + input_height, input_width = get_image_size(input_image) + min_size, max_size = shorter, longer + + scale = min_size / min(input_height, input_width) + + if input_height < input_width: + new_height = min_size + new_width = scale * input_width + else: + new_height = scale * input_height + new_width = min_size + + if max(new_height, new_width) > max_size: + scale = max_size / max(new_height, new_width) + new_height = scale * new_height + new_width = scale * new_width + + new_height, new_width = int(new_height + 0.5), int(new_width + 0.5) + new_height = new_height // size_divisor * size_divisor + new_width = new_width // size_divisor * size_divisor + + return new_height, new_width + + +class BlipFeatureExtractor(BaseImageProcessor): + r""" + Constructs a ViLT image processor. + + Args: + do_resize (`bool`, *optional*, defaults to `True`): + Whether to resize the image's (height, width) dimensions to the specified `size`. Can be overridden by the + `do_resize` parameter in the `preprocess` method. + size (`Dict[str, int]` *optional*, defaults to `{"shortest_edge": 384}`): + Resize the shorter side of the input to `size["shortest_edge"]`. The longer side will be limited to under + `int((1333 / 800) * size["shortest_edge"])` while preserving the aspect ratio. Only has an effect if + `do_resize` is set to `True`. Can be overridden by the `size` parameter in the `preprocess` method. + size_divisor (`int`, *optional*, defaults to 32): + The size by which to make sure both the height and width can be divided. Only has an effect if `do_resize` + is set to `True`. Can be overridden by the `size_divisor` parameter in the `preprocess` method. + resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`): + Resampling filter to use if resizing the image. Only has an effect if `do_resize` is set to `True`. Can be + overridden by the `resample` parameter in the `preprocess` method. + do_rescale (`bool`, *optional*, defaults to `True`): + Wwhether to rescale the image by the specified scale `rescale_factor`. Can be overridden by the + `do_rescale` parameter in the `preprocess` method. + rescale_factor (`int` or `float`, *optional*, defaults to `1/255`): + Scale factor to use if rescaling the image. Only has an effect if `do_rescale` is set to `True`. Can be + overridden by the `rescale_factor` parameter in the `preprocess` method. + do_normalize (`bool`, *optional*, defaults to `True`): + Whether to normalize the image. Can be overridden by the `do_normalize` parameter in the `preprocess` + method. Can be overridden by the `do_normalize` parameter in the `preprocess` method. + image_mean (`float` or `List[float]`, *optional*, defaults to `IMAGENET_STANDARD_MEAN`): + Mean to use if normalizing the image. This is a float or list of floats the length of the number of + channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method. Can be + overridden by the `image_mean` parameter in the `preprocess` method. + image_std (`float` or `List[float]`, *optional*, defaults to `IMAGENET_STANDARD_STD`): + Standard deviation to use if normalizing the image. This is a float or list of floats the length of the + number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method. + Can be overridden by the `image_std` parameter in the `preprocess` method. + do_pad (`bool`, *optional*, defaults to `True`): + Whether to pad the image to the `(max_height, max_width)` of the images in the batch. Can be overridden by + the `do_pad` parameter in the `preprocess` method. + """ + + model_input_names = ["pixel_values"] + + def __init__( + self, + do_resize: bool = True, + size: Dict[str, int] = None, + size_divisor: int = 32, + resample: PILImageResampling = PILImageResampling.BICUBIC, + do_rescale: bool = True, + rescale_factor: Union[int, float] = 1 / 255, + do_normalize: bool = True, + image_mean: Optional[Union[float, List[float]]] = None, + image_std: Optional[Union[float, List[float]]] = None, + do_pad: bool = True, + **kwargs + ) -> None: + if "pad_and_return_pixel_mask" in kwargs: + do_pad = kwargs.pop("pad_and_return_pixel_mask") + + super().__init__(**kwargs) + size = size if size is not None else {"shortest_edge": 384} + size = get_size_dict(size, default_to_square=False) + + self.do_resize = do_resize + self.size = size + self.size_divisor = size_divisor + self.resample = resample + self.do_rescale = do_rescale + self.rescale_factor = rescale_factor + self.do_normalize = do_normalize + self.image_mean = image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN + self.image_std = image_std if image_std is not None else IMAGENET_STANDARD_STD + self.do_pad = do_pad + + def resize( + self, + image: np.ndarray, + size: Dict[str, int], + size_divisor: int = 32, + resample: PILImageResampling = PILImageResampling.BICUBIC, + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs + ) -> np.ndarray: + """ + Resize an image. + + Resizes the shorter side of the image to `size["shortest_edge"]` while preserving the aspect ratio. If the + longer side is larger than the max size `(int(`size["shortest_edge"]` * 1333 / 800))`, the longer side is then + resized to the max size while preserving the aspect ratio. + + Args: + image (`np.ndarray`): + Image to resize. + size (`Dict[str, int]`): + Controls the size of the output image. Should be of the form `{"shortest_edge": int}`. + size_divisor (`int`, defaults to 32): + The image is resized to a size that is a multiple of this value. + resample (`PILImageResampling` filter, *optional*, defaults to `PILImageResampling.BICUBIC`): + Resampling filter to use when resiizing the image. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format of the image. If not provided, it will be the same as the input image. + """ + size = get_size_dict(size, default_to_square=False) + if "shortest_edge" not in size: + raise ValueError(f"The `size` dictionary must contain the key `shortest_edge`. Got {size.keys()}") + shorter = size["shortest_edge"] + longer = int(1333 / 800 * shorter) + output_size = get_resize_output_image_size(image, shorter=shorter, longer=longer, size_divisor=size_divisor) + return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) + + def rescale( + self, + image: np.ndarray, + scale: Union[int, float], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs + ): + """ + Rescale an image by a scale factor. image = image * scale. + + Args: + image (`np.ndarray`): + Image to rescale. + scale (`int` or `float`): + Scale to apply to the image. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format of the image. If not provided, it will be the same as the input image. + """ + return rescale(image, scale=scale, data_format=data_format, **kwargs) + + def normalize( + self, + image: np.ndarray, + mean: Union[float, List[float]], + std: Union[float, List[float]], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs + ) -> np.ndarray: + """ + Normalize an image. image = (image - image_mean) / image_std. + + Args: + image (`np.ndarray`): + Image to normalize. + mean (`float` or `List[float]`): + Image mean. + std (`float` or `List[float]`): + Image standard deviation. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format of the image. If not provided, it will be the same as the input image. + """ + return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) + + def pad( + self, + images: List[np.ndarray], + return_pixel_mask: bool = True, + return_tensors: Optional[Union[str, TensorType]] = None, + data_format: Optional[ChannelDimension] = None, + ) -> BatchFeature: + """ + Pads a batch of images with zeros to the size of largest height and width in the batch and optionally returns + their corresponding pixel mask. + + Args: + images (`List[np.ndarray]`): + Batch of images to pad. + return_pixel_mask (`bool`, *optional*, defaults to `False`): + Whether to return the pixel mask. + return_tensors (`str` or `TensorType`, *optional*): + The type of tensors to return. Can be one of: + - Unset: Return a list of `np.ndarray`. + - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`. + - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. + - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. + - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format of the image. If not provided, it will be the same as the input image. + """ + pad_size = get_max_dimensions(images) + padded_images = [pad(image=image, output_size=pad_size, data_format=data_format) for image in images] + data = {"pixel_values": padded_images} + if return_pixel_mask: + masks = [make_pixel_mask(image=image, output_size=pad_size) for image in images] + data["pixel_mask"] = masks + + return BatchFeature(data=data, tensor_type=return_tensors) + + def pad_and_create_pixel_mask( + self, + pixel_values_list: List[ImageInput], + return_tensors: Optional[Union[str, TensorType]] = None, + data_format: Optional[ChannelDimension] = None, + ) -> BatchFeature: + """ + Pads a batch of images with zeros to the size of largest height and width in the batch and returns their + corresponding pixel mask. + + Args: + images (`List[np.ndarray]`): + Batch of images to pad. + return_tensors (`str` or `TensorType`, *optional*): + The type of tensors to return. Can be one of: + - Unset: Return a list of `np.ndarray`. + - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`. + - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. + - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. + - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format of the image. If not provided, it will be the same as the input image. + """ + warnings.warn( + "This method is deprecated and will be removed in v4.26.0. Please use pad instead.", FutureWarning + ) + # pad expects a list of np.ndarray, but the previous feature extractors expected torch tensors + images = [to_numpy_array(image) for image in pixel_values_list] + return self.pad( + images=images, + return_pixel_mask=True, + return_tensors=return_tensors, + data_format=data_format, + ) + + def preprocess( + self, + images: ImageInput, + do_resize: Optional[bool] = None, + size: Optional[Dict[str, int]] = None, + size_divisor: Optional[int] = None, + resample: PILImageResampling = None, + do_rescale: Optional[bool] = None, + rescale_factor: Optional[float] = None, + do_normalize: Optional[bool] = None, + image_mean: Optional[Union[float, List[float]]] = None, + image_std: Optional[Union[float, List[float]]] = None, + do_pad: Optional[bool] = None, + return_tensors: Optional[Union[str, TensorType]] = None, + data_format: ChannelDimension = ChannelDimension.FIRST, + **kwargs, + ) -> PIL.Image.Image: + """ + Preprocess an image or batch of images. + + Args: + images (`ImageInput`): + Image to preprocess. + do_resize (`bool`, *optional*, defaults to `self.do_resize`): + Whether to resize the image. + size (`Dict[str, int]`, *optional*, defaults to `self.size`): + Controls the size of the image after `resize`. The shortest edge of the image is resized to + `size["shortest_edge"]` whilst preserving the aspect ratio. If the longest edge of this resized image + is > `int(size["shortest_edge"] * (1333 / 800))`, then the image is resized again to make the longest + edge equal to `int(size["shortest_edge"] * (1333 / 800))`. + size_divisor (`int`, *optional*, defaults to `self.size_divisor`): + The image is resized to a size that is a multiple of this value. + resample (`PILImageResampling`, *optional*, defaults to `self.resample`): + Resampling filter to use if resizing the image. Only has an effect if `do_resize` is set to `True`. + do_rescale (`bool`, *optional*, defaults to `self.do_rescale`): + Whether to rescale the image values between [0 - 1]. + rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`): + Rescale factor to rescale the image by if `do_rescale` is set to `True`. + do_normalize (`bool`, *optional*, defaults to `self.do_normalize`): + Whether to normalize the image. + image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`): + Image mean to normalize the image by if `do_normalize` is set to `True`. + image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`): + Image standard deviation to normalize the image by if `do_normalize` is set to `True`. + do_pad (`bool`, *optional*, defaults to `self.do_pad`): + Whether to pad the image to the (max_height, max_width) in the batch. If `True`, a pixel mask is also + created and returned. + return_tensors (`str` or `TensorType`, *optional*): + The type of tensors to return. Can be one of: + - Unset: Return a list of `np.ndarray`. + - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`. + - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. + - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. + - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. + data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): + The channel dimension format for the output image. Can be one of: + - `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `ChannelDimension.LAST`: image in (height, width, num_channels) format. + """ + do_resize = do_resize if do_resize is not None else self.do_resize + size_divisor = size_divisor if size_divisor is not None else self.size_divisor + resample = resample if resample is not None else self.resample + do_rescale = do_rescale if do_rescale is not None else self.do_rescale + rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor + do_normalize = do_normalize if do_normalize is not None else self.do_normalize + image_mean = image_mean if image_mean is not None else self.image_mean + image_std = image_std if image_std is not None else self.image_std + do_pad = do_pad if do_pad is not None else self.do_pad + + size = size if size is not None else self.size + size = get_size_dict(size, default_to_square=False) + + if not is_batched(images): + images = [images] + + if not valid_images(images): + raise ValueError( + "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, " + "torch.Tensor, tf.Tensor or jax.ndarray." + ) + + if do_resize and size is None or resample is None: + raise ValueError("Size and resample must be specified if do_resize is True.") + + if do_rescale and rescale_factor is None: + raise ValueError("Rescale factor must be specified if do_rescale is True.") + + if do_normalize and (image_mean is None or image_std is None): + raise ValueError("Image mean and std must be specified if do_normalize is True.") + + # All transformations expect numpy arrays. + images = [to_numpy_array(image) for image in images] + + if do_resize: + images = [ + self.resize(image=image, size=size, size_divisor=size_divisor, resample=resample) for image in images + ] + + if do_rescale: + images = [self.rescale(image=image, scale=rescale_factor) for image in images] + + if do_normalize: + images = [self.normalize(image=image, mean=image_mean, std=image_std) for image in images] + + images = [to_channel_dimension_format(image, data_format) for image in images] + + if do_pad: + encoded_outputs = self.pad(images, return_pixel_mask=True, return_tensors=return_tensors) + else: + encoded_outputs = BatchFeature(data={"pixel_values": images}, tensor_type=return_tensors) + + return encoded_outputs diff --git a/src/transformers/models/blip/processing_blip.py b/src/transformers/models/blip/processing_blip.py new file mode 100644 index 000000000000..04972ec9661f --- /dev/null +++ b/src/transformers/models/blip/processing_blip.py @@ -0,0 +1,114 @@ +# coding=utf-8 +# Copyright 2022 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Processor class for Blip. +""" + +from typing import List, Optional, Union + +from ...processing_utils import ProcessorMixin +from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy +from ...utils import TensorType + + +class BlipProcessor(ProcessorMixin): + r""" + Constructs a Blip processor which wraps a BERT tokenizer and Blip feature extractor into a single processor. + + [`BlipProcessor`] offers all the functionalities of [`BlipFeatureExtractor`] and [`BertTokenizerFast`]. See the + docstring of [`~BlipProcessor.__call__`] and [`~BlipProcessor.decode`] for more information. + + Args: + feature_extractor (`BlipFeatureExtractor`): + An instance of [`BlipFeatureExtractor`]. The feature extractor is a required input. + tokenizer (`BertTokenizerFast`): + An instance of ['BertTokenizerFast`]. The tokenizer is a required input. + """ + feature_extractor_class = "BlipFeatureExtractor" + tokenizer_class = ("BertTokenizer", "BertTokenizerFast") + + def __init__(self, feature_extractor, tokenizer): + super().__init__(feature_extractor, tokenizer) + self.current_processor = self.feature_extractor + + def __call__( + self, + images, + text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None, + add_special_tokens: bool = True, + padding: Union[bool, str, PaddingStrategy] = False, + truncation: Union[bool, str, TruncationStrategy] = None, + max_length: Optional[int] = None, + stride: int = 0, + pad_to_multiple_of: Optional[int] = None, + return_token_type_ids: Optional[bool] = None, + return_attention_mask: Optional[bool] = None, + return_overflowing_tokens: bool = False, + return_special_tokens_mask: bool = False, + return_offsets_mapping: bool = False, + return_length: bool = False, + verbose: bool = True, + return_tensors: Optional[Union[str, TensorType]] = None, + **kwargs + ) -> BatchEncoding: + """ + This method uses [`BlipFeatureExtractor.__call__`] method to prepare image(s) for the model, and + [`BertTokenizerFast.__call__`] to prepare text for the model. + + Please refer to the docstring of the above two methods for more information. + """ + encoding = self.tokenizer( + text=text, + add_special_tokens=add_special_tokens, + padding=padding, + truncation=truncation, + max_length=max_length, + stride=stride, + pad_to_multiple_of=pad_to_multiple_of, + return_token_type_ids=return_token_type_ids, + return_attention_mask=return_attention_mask, + return_overflowing_tokens=return_overflowing_tokens, + return_special_tokens_mask=return_special_tokens_mask, + return_offsets_mapping=return_offsets_mapping, + return_length=return_length, + verbose=verbose, + return_tensors=return_tensors, + **kwargs, + ) + # add pixel_values + pixel_mask + encoding_feature_extractor = self.feature_extractor(images, return_tensors=return_tensors) + encoding.update(encoding_feature_extractor) + + return encoding + + def batch_decode(self, *args, **kwargs): + """ + This method forwards all its arguments to BertTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please + refer to the docstring of this method for more information. + """ + return self.tokenizer.batch_decode(*args, **kwargs) + + def decode(self, *args, **kwargs): + """ + This method forwards all its arguments to BertTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to + the docstring of this method for more information. + """ + return self.tokenizer.decode(*args, **kwargs) + + @property + def model_input_names(self): + tokenizer_input_names = self.tokenizer.model_input_names + feature_extractor_input_names = self.feature_extractor.model_input_names + return list(dict.fromkeys(tokenizer_input_names + feature_extractor_input_names)) From fb45b64748c806acb07c2b5a4d6805d52105bceb Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Sun, 11 Dec 2022 13:04:02 +0000 Subject: [PATCH 14/96] fix processors --- docs/source/en/model_doc/blip.mdx | 13 +++++++++++++ src/transformers/__init__.py | 11 ++++++++++- src/transformers/models/blip/__init__.py | 4 ++-- .../models/blip/feature_extraction_blip.py | 2 -- src/transformers/utils/dummy_vision_objects.py | 14 ++++++++++++++ 5 files changed, 39 insertions(+), 5 deletions(-) diff --git a/docs/source/en/model_doc/blip.mdx b/docs/source/en/model_doc/blip.mdx index 83144545b6eb..1e1bd7e98a6d 100644 --- a/docs/source/en/model_doc/blip.mdx +++ b/docs/source/en/model_doc/blip.mdx @@ -72,7 +72,20 @@ The original code can be found [here](). [[autodoc]] BlipForImageTextRetrieval - forward + ## BlipForVisualQuestionAnswering [[autodoc]] BlipForVisualQuestionAnswering - forward + + +## BlipProcessor + +[[autodoc]] BlipProcessor + - forward + + +## BlipFeatureExtractor + +[[autodoc]] BlipFeatureExtractor + - forward \ No newline at end of file diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 0ac684cb99ec..36fe06b93db6 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -171,6 +171,7 @@ "models.blip": [ "BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP", "BlipConfig", + "BlipProcessor", "BlipTextConfig", "BlipVisionConfig", ], @@ -754,6 +755,7 @@ _import_structure["image_utils"] = ["ImageFeatureExtractionMixin"] _import_structure["models.beit"].extend(["BeitFeatureExtractor", "BeitImageProcessor"]) _import_structure["models.bit"].extend(["BitImageProcessor"]) + _import_structure["models.blip"].extend(["BlipFeatureExtractor", "BlipProcessor"]) _import_structure["models.chinese_clip"].extend(["ChineseCLIPFeatureExtractor", "ChineseCLIPImageProcessor"]) _import_structure["models.clip"].extend(["CLIPFeatureExtractor", "CLIPImageProcessor"]) _import_structure["models.conditional_detr"].extend( @@ -3450,7 +3452,13 @@ BlenderbotSmallConfig, BlenderbotSmallTokenizer, ) - from .models.blip import BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP, BlipConfig, BlipTextConfig, BlipVisionConfig + from .models.blip import ( + BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP, + BlipConfig, + BlipProcessor, + BlipTextConfig, + BlipVisionConfig, + ) from .models.bloom import BLOOM_PRETRAINED_CONFIG_ARCHIVE_MAP, BloomConfig from .models.byt5 import ByT5Tokenizer from .models.camembert import CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, CamembertConfig @@ -3963,6 +3971,7 @@ from .image_utils import ImageFeatureExtractionMixin from .models.beit import BeitFeatureExtractor, BeitImageProcessor from .models.bit import BitImageProcessor + from .models.blip import BlipFeatureExtractor, BlipProcessor from .models.chinese_clip import ChineseCLIPFeatureExtractor, ChineseCLIPImageProcessor from .models.clip import CLIPFeatureExtractor, CLIPImageProcessor from .models.conditional_detr import ConditionalDetrFeatureExtractor, ConditionalDetrImageProcessor diff --git a/src/transformers/models/blip/__init__.py b/src/transformers/models/blip/__init__.py index 39e8e124a090..f9fae1a5c070 100644 --- a/src/transformers/models/blip/__init__.py +++ b/src/transformers/models/blip/__init__.py @@ -35,8 +35,8 @@ except OptionalDependencyNotAvailable: pass else: - _import_structure["feature_extraction_vilt"] = ["BlipFeatureExtractor"] - _import_structure["processing_vilt"] = ["BlipProcessor"] + _import_structure["feature_extraction_blip"] = ["BlipFeatureExtractor"] + _import_structure["processing_blip"] = ["BlipProcessor"] try: diff --git a/src/transformers/models/blip/feature_extraction_blip.py b/src/transformers/models/blip/feature_extraction_blip.py index 52ba3ee49e33..155d22e1b461 100644 --- a/src/transformers/models/blip/feature_extraction_blip.py +++ b/src/transformers/models/blip/feature_extraction_blip.py @@ -14,8 +14,6 @@ # limitations under the License. """Feature extractor class for ViLT.""" -from ...utils import logging - import warnings from typing import Any, Dict, Iterable, List, Optional, Tuple, Union diff --git a/src/transformers/utils/dummy_vision_objects.py b/src/transformers/utils/dummy_vision_objects.py index 1aa5e29a7d3a..ce445b380326 100644 --- a/src/transformers/utils/dummy_vision_objects.py +++ b/src/transformers/utils/dummy_vision_objects.py @@ -50,6 +50,20 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["vision"]) +class BlipFeatureExtractor(metaclass=DummyObject): + _backends = ["vision"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["vision"]) + + +class BlipProcessor(metaclass=DummyObject): + _backends = ["vision"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["vision"]) + + class ChineseCLIPFeatureExtractor(metaclass=DummyObject): _backends = ["vision"] From d256a0a202ba1e098a67ad86b82927520d3b392b Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Sun, 11 Dec 2022 13:25:46 +0000 Subject: [PATCH 15/96] fix doc --- docs/source/en/model_doc/blip.mdx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/source/en/model_doc/blip.mdx b/docs/source/en/model_doc/blip.mdx index 1e1bd7e98a6d..c156519b9ff1 100644 --- a/docs/source/en/model_doc/blip.mdx +++ b/docs/source/en/model_doc/blip.mdx @@ -82,10 +82,9 @@ The original code can be found [here](). ## BlipProcessor [[autodoc]] BlipProcessor - - forward ## BlipFeatureExtractor [[autodoc]] BlipFeatureExtractor - - forward \ No newline at end of file + - preprocess \ No newline at end of file From 980b72320d2f48f3507d82702e1f7cffa96024c2 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Sun, 11 Dec 2022 13:29:16 +0000 Subject: [PATCH 16/96] fill title --- README.md | 2 +- README_es.md | 2 +- README_hd.md | 2 +- README_ja.md | 2 +- README_ko.md | 2 +- README_zh-hans.md | 2 +- README_zh-hant.md | 2 +- docs/source/en/index.mdx | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index adc84d175c8e..2378675fc718 100644 --- a/README.md +++ b/README.md @@ -277,7 +277,7 @@ Current number of checkpoints: ![](https://img.shields.io/endpoint?url=https://h 1. **[BiT](https://huggingface.co/docs/transformers/main/model_doc/bit)** (from Google AI) released with the paper [Big Transfer (BiT): General Visual Representation Learning](https://arxiv.org/abs/1912.11370) by Alexander Kolesnikov, Lucas Beyer, Xiaohua Zhai, Joan Puigcerver, Jessica Yung, Sylvain Gelly, Neil Houlsby. 1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. 1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. -1. **[BLIP](https://huggingface.co/docs/transformers/main/model_doc/blip)** (from ) released with the paper []() by . +1. **[BLIP](https://huggingface.co/docs/transformers/main/model_doc/blip)** (from Salesforce) released with the paper [BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation](https://arxiv.org/abs/2201.12086) by Junnan Li, Dongxu Li, Caiming Xiong, Steven Hoi. 1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (from BigScience workshop) released by the [BigScience Workshop](https://bigscience.huggingface.co/). 1. **[BORT](https://huggingface.co/docs/transformers/model_doc/bort)** (from Alexa) released with the paper [Optimal Subarchitecture Extraction For BERT](https://arxiv.org/abs/2010.10499) by Adrian de Wynter and Daniel J. Perry. 1. **[ByT5](https://huggingface.co/docs/transformers/model_doc/byt5)** (from Google Research) released with the paper [ByT5: Towards a token-free future with pre-trained byte-to-byte models](https://arxiv.org/abs/2105.13626) by Linting Xue, Aditya Barua, Noah Constant, Rami Al-Rfou, Sharan Narang, Mihir Kale, Adam Roberts, Colin Raffel. diff --git a/README_es.md b/README_es.md index 85e17802fc6e..e0e72fe8651f 100644 --- a/README_es.md +++ b/README_es.md @@ -277,7 +277,7 @@ Número actual de puntos de control: ![](https://img.shields.io/endpoint?url=htt 1. **[BiT](https://huggingface.co/docs/transformers/main/model_doc/bit)** (from Google AI) released with the paper [Big Transfer (BiT) by Alexander Kolesnikov, Lucas Beyer, Xiaohua Zhai, Joan Puigcerver, Jessica Yung, Sylvain Gelly, Neil Houlsby. 1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. 1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. -1. **[BLIP](https://huggingface.co/docs/transformers/main/model_doc/blip)** (from ) released with the paper []() by . +1. **[BLIP](https://huggingface.co/docs/transformers/main/model_doc/blip)** (from Salesforce) released with the paper [BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation](https://arxiv.org/abs/2201.12086) by Junnan Li, Dongxu Li, Caiming Xiong, Steven Hoi. 1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (from BigScience workshop) released by the [BigScience Workshop](https://bigscience.huggingface.co/). 1. **[BORT](https://huggingface.co/docs/transformers/model_doc/bort)** (from Alexa) released with the paper [Optimal Subarchitecture Extraction For BERT](https://arxiv.org/abs/2010.10499) by Adrian de Wynter and Daniel J. Perry. 1. **[ByT5](https://huggingface.co/docs/transformers/model_doc/byt5)** (from Google Research) released with the paper [ByT5: Towards a token-free future with pre-trained byte-to-byte models](https://arxiv.org/abs/2105.13626) by Linting Xue, Aditya Barua, Noah Constant, Rami Al-Rfou, Sharan Narang, Mihir Kale, Adam Roberts, Colin Raffel. diff --git a/README_hd.md b/README_hd.md index c64e4c20a0bf..a514fa698a3e 100644 --- a/README_hd.md +++ b/README_hd.md @@ -250,7 +250,7 @@ conda install -c huggingface transformers 1. **[BiT](https://huggingface.co/docs/transformers/main/model_doc/bit)** (from Google AI) released with the paper [Big Transfer (BiT) by Alexander Kolesnikov, Lucas Beyer, Xiaohua Zhai, Joan Puigcerver, Jessica Yung, Sylvain Gelly, Neil Houlsby. 1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (फेसबुक से) साथ में कागज [एक ओपन-डोमेन चैटबॉट बनाने की विधि](https://arxiv.org /abs/2004.13637) स्टीफन रोलर, एमिली दीनन, नमन गोयल, दा जू, मैरी विलियमसन, यिनहान लियू, जिंग जू, मायल ओट, कर्ट शस्टर, एरिक एम। स्मिथ, वाई-लैन बॉरो, जेसन वेस्टन द्वारा। 1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (फेसबुक से) साथ में पेपर [एक ओपन-डोमेन चैटबॉट बनाने की रेसिपी](https://arxiv .org/abs/2004.13637) स्टीफन रोलर, एमिली दीनन, नमन गोयल, दा जू, मैरी विलियमसन, यिनहान लियू, जिंग जू, मायल ओट, कर्ट शस्टर, एरिक एम स्मिथ, वाई-लैन बॉरो, जेसन वेस्टन द्वारा। -1. **[BLIP](https://huggingface.co/docs/transformers/main/model_doc/blip)** (from ) released with the paper []() by . +1. **[BLIP](https://huggingface.co/docs/transformers/main/model_doc/blip)** (from Salesforce) released with the paper [BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation](https://arxiv.org/abs/2201.12086) by Junnan Li, Dongxu Li, Caiming Xiong, Steven Hoi. 1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (from BigScience workshop) released by the [BigSicence Workshop](https://bigscience.huggingface.co/). 1. **[BORT](https://huggingface.co/docs/transformers/model_doc/bort)** (एलेक्सा से) कागज के साथ [बीईआरटी के लिए ऑप्टिमल सबआर्किटेक्चर एक्सट्रैक्शन](https://arxiv.org/abs/ 2010.10499) एड्रियन डी विंटर और डैनियल जे पेरी द्वारा। 1. **[ByT5](https://huggingface.co/docs/transformers/model_doc/byt5)** (Google अनुसंधान से) साथ में कागज [ByT5: पूर्व-प्रशिक्षित बाइट-टू-बाइट मॉडल के साथ एक टोकन-मुक्त भविष्य की ओर] (https://arxiv.org/abs/2105.13626) Linting Xue, Aditya Barua, Noah Constant, रामी अल-रफू, शरण नारंग, मिहिर काले, एडम रॉबर्ट्स, कॉलिन रैफेल द्वारा पोस्ट किया गया। diff --git a/README_ja.md b/README_ja.md index 1bc6bbcd3d77..777e7a74b262 100644 --- a/README_ja.md +++ b/README_ja.md @@ -312,7 +312,7 @@ Flax、PyTorch、TensorFlowをcondaでインストールする方法は、それ 1. **[BiT](https://huggingface.co/docs/transformers/main/model_doc/bit)** (from Google AI) released with the paper [Big Transfer (BiT) by Alexander Kolesnikov, Lucas Beyer, Xiaohua Zhai, Joan Puigcerver, Jessica Yung, Sylvain Gelly, Neil Houlsby. 1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. 1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. -1. **[BLIP](https://huggingface.co/docs/transformers/main/model_doc/blip)** (from ) released with the paper []() by . +1. **[BLIP](https://huggingface.co/docs/transformers/main/model_doc/blip)** (from Salesforce) released with the paper [BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation](https://arxiv.org/abs/2201.12086) by Junnan Li, Dongxu Li, Caiming Xiong, Steven Hoi. 1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (from BigScience workshop) released by the [BigScience Workshop](https://bigscience.huggingface.co/). 1. **[BORT](https://huggingface.co/docs/transformers/model_doc/bort)** (from Alexa) released with the paper [Optimal Subarchitecture Extraction For BERT](https://arxiv.org/abs/2010.10499) by Adrian de Wynter and Daniel J. Perry. 1. **[ByT5](https://huggingface.co/docs/transformers/model_doc/byt5)** (from Google Research) released with the paper [ByT5: Towards a token-free future with pre-trained byte-to-byte models](https://arxiv.org/abs/2105.13626) by Linting Xue, Aditya Barua, Noah Constant, Rami Al-Rfou, Sharan Narang, Mihir Kale, Adam Roberts, Colin Raffel. diff --git a/README_ko.md b/README_ko.md index 5338db7e8b8c..ac94ad16dc90 100644 --- a/README_ko.md +++ b/README_ko.md @@ -227,7 +227,7 @@ Flax, PyTorch, TensorFlow 설치 페이지에서 이들을 conda로 설치하는 1. **[BiT](https://huggingface.co/docs/transformers/main/model_doc/bit)** (from Google AI) released with the paper [Big Transfer (BiT) by Alexander Kolesnikov, Lucas Beyer, Xiaohua Zhai, Joan Puigcerver, Jessica Yung, Sylvain Gelly, Neil Houlsby. 1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. 1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. -1. **[BLIP](https://huggingface.co/docs/transformers/main/model_doc/blip)** (from ) released with the paper []() by . +1. **[BLIP](https://huggingface.co/docs/transformers/main/model_doc/blip)** (from Salesforce) released with the paper [BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation](https://arxiv.org/abs/2201.12086) by Junnan Li, Dongxu Li, Caiming Xiong, Steven Hoi. 1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (from BigScience workshop) released by the [BigScience Workshop](https://bigscience.huggingface.co/). 1. **[BORT](https://huggingface.co/docs/transformers/model_doc/bort)** (from Alexa) released with the paper [Optimal Subarchitecture Extraction For BERT](https://arxiv.org/abs/2010.10499) by Adrian de Wynter and Daniel J. Perry. 1. **[ByT5](https://huggingface.co/docs/transformers/model_doc/byt5)** (from Google Research) released with the paper [ByT5: Towards a token-free future with pre-trained byte-to-byte models](https://arxiv.org/abs/2105.13626) by Linting Xue, Aditya Barua, Noah Constant, Rami Al-Rfou, Sharan Narang, Mihir Kale, Adam Roberts, Colin Raffel. diff --git a/README_zh-hans.md b/README_zh-hans.md index 0ba96c44e093..2322b2cd808b 100644 --- a/README_zh-hans.md +++ b/README_zh-hans.md @@ -251,7 +251,7 @@ conda install -c huggingface transformers 1. **[BiT](https://huggingface.co/docs/transformers/main/model_doc/bit)** (来自 Google AI) 伴随论文 [Big Transfer (BiT) 由 Alexander Kolesnikov, Lucas Beyer, Xiaohua Zhai, Joan Puigcerver, Jessica Yung, Sylvain Gelly, Neil Houlsby 发布。 1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (来自 Facebook) 伴随论文 [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) 由 Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston 发布。 1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (来自 Facebook) 伴随论文 [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) 由 Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston 发布。 -1. **[BLIP](https://huggingface.co/docs/transformers/main/model_doc/blip)** (from ) released with the paper []() by . +1. **[BLIP](https://huggingface.co/docs/transformers/main/model_doc/blip)** (来自 Salesforce) 伴随论文 [BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation](https://arxiv.org/abs/2201.12086) 由 Junnan Li, Dongxu Li, Caiming Xiong, Steven Hoi 发布。 1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (from BigScience workshop) released by the [BigScience Workshop](https://bigscience.huggingface.co/). 1. **[BORT](https://huggingface.co/docs/transformers/model_doc/bort)** (来自 Alexa) 伴随论文 [Optimal Subarchitecture Extraction For BERT](https://arxiv.org/abs/2010.10499) 由 Adrian de Wynter and Daniel J. Perry 发布。 1. **[ByT5](https://huggingface.co/docs/transformers/model_doc/byt5)** (来自 Google Research) 伴随论文 [ByT5: Towards a token-free future with pre-trained byte-to-byte models](https://arxiv.org/abs/2105.13626) 由 Linting Xue, Aditya Barua, Noah Constant, Rami Al-Rfou, Sharan Narang, Mihir Kale, Adam Roberts, Colin Raffel 发布。 diff --git a/README_zh-hant.md b/README_zh-hant.md index 63547dc647d9..dbb2f215b1fa 100644 --- a/README_zh-hant.md +++ b/README_zh-hant.md @@ -263,7 +263,7 @@ conda install -c huggingface transformers 1. **[BiT](https://huggingface.co/docs/transformers/main/model_doc/bit)** (from Google AI) released with the paper [Big Transfer (BiT) by Alexander Kolesnikov, Lucas Beyer, Xiaohua Zhai, Joan Puigcerver, Jessica Yung, Sylvain Gelly, Neil Houlsby. 1. **[Blenderbot](https://huggingface.co/docs/transformers/model_doc/blenderbot)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. 1. **[BlenderbotSmall](https://huggingface.co/docs/transformers/model_doc/blenderbot-small)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. -1. **[BLIP](https://huggingface.co/docs/transformers/main/model_doc/blip)** (from ) released with the paper []() by . +1. **[BLIP](https://huggingface.co/docs/transformers/main/model_doc/blip)** (from Salesforce) released with the paper [BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation](https://arxiv.org/abs/2201.12086) by Junnan Li, Dongxu Li, Caiming Xiong, Steven Hoi. 1. **[BLOOM](https://huggingface.co/docs/transformers/model_doc/bloom)** (from BigScience workshop) released by the [BigScience Workshop](https://bigscience.huggingface.co/). 1. **[BORT](https://huggingface.co/docs/transformers/model_doc/bort)** (from Alexa) released with the paper [Optimal Subarchitecture Extraction For BERT](https://arxiv.org/abs/2010.10499) by Adrian de Wynter and Daniel J. Perry. 1. **[ByT5](https://huggingface.co/docs/transformers/model_doc/byt5)** (from Google Research) released with the paper [ByT5: Towards a token-free future with pre-trained byte-to-byte models](https://arxiv.org/abs/2105.13626) by Linting Xue, Aditya Barua, Noah Constant, Rami Al-Rfou, Sharan Narang, Mihir Kale, Adam Roberts, Colin Raffel. diff --git a/docs/source/en/index.mdx b/docs/source/en/index.mdx index dbd9a5ac090f..2e2fa6514a34 100644 --- a/docs/source/en/index.mdx +++ b/docs/source/en/index.mdx @@ -64,7 +64,7 @@ The documentation is organized into five sections: 1. **[BiT](model_doc/bit)** (from Google AI) released with the paper [Big Transfer (BiT): General Visual Representation Learning](https://arxiv.org/abs/1912.11370) by Alexander Kolesnikov, Lucas Beyer, Xiaohua Zhai, Joan Puigcerver, Jessica Yung, Sylvain Gelly, Neil Houlsby. 1. **[Blenderbot](model_doc/blenderbot)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. 1. **[BlenderbotSmall](model_doc/blenderbot-small)** (from Facebook) released with the paper [Recipes for building an open-domain chatbot](https://arxiv.org/abs/2004.13637) by Stephen Roller, Emily Dinan, Naman Goyal, Da Ju, Mary Williamson, Yinhan Liu, Jing Xu, Myle Ott, Kurt Shuster, Eric M. Smith, Y-Lan Boureau, Jason Weston. -1. **[BLIP](model_doc/blip)** (from ) released with the paper []() by . +1. **[BLIP](model_doc/blip)** (from Salesforce) released with the paper [BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation](https://arxiv.org/abs/2201.12086) by Junnan Li, Dongxu Li, Caiming Xiong, Steven Hoi. 1. **[BLOOM](model_doc/bloom)** (from BigScience workshop) released by the [BigScience Workshop](https://bigscience.huggingface.co/). 1. **[BORT](model_doc/bort)** (from Alexa) released with the paper [Optimal Subarchitecture Extraction For BERT](https://arxiv.org/abs/2010.10499) by Adrian de Wynter and Daniel J. Perry. 1. **[ByT5](model_doc/byt5)** (from Google Research) released with the paper [ByT5: Towards a token-free future with pre-trained byte-to-byte models](https://arxiv.org/abs/2105.13626) by Linting Xue, Aditya Barua, Noah Constant, Rami Al-Rfou, Sharan Narang, Mihir Kale, Adam Roberts, Colin Raffel. From e53ad6cc3ffac2c39d7cf5fb66be7a45fc4a3ed7 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Sun, 11 Dec 2022 13:33:17 +0000 Subject: [PATCH 17/96] add content doc --- docs/source/en/model_doc/blip.mdx | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/source/en/model_doc/blip.mdx b/docs/source/en/model_doc/blip.mdx index c156519b9ff1..d8a2d9a53d3f 100644 --- a/docs/source/en/model_doc/blip.mdx +++ b/docs/source/en/model_doc/blip.mdx @@ -14,19 +14,20 @@ specific language governing permissions and limitations under the License. ## Overview -The BLIP model was proposed in []() by . - +The BLIP model was proposed in [BLIP: Bootstrapping Language-Image Pre-training for Unified Vision-Language Understanding and Generation](https://arxiv.org/abs/2201.12086) by Junnan Li, Dongxu Li, Caiming Xiong, Steven Hoi. -The abstract from the paper is the following: - -** +BLIP is a model that is able to perform various multi-modal tasks including +- Visual Question Answering +- Image-Text retrieval (Image-text matching) +- Image Captioning -Tips: +The abstract from the paper is the following: - +*Vision-Language Pre-training (VLP) has advanced the performance for many vision-language tasks. +However, most existing pre-trained models only excel in either understanding-based tasks or generation-based tasks. Furthermore, performance improvement has been largely achieved by scaling up the dataset with noisy image-text pairs collected from the web, which is a suboptimal source of supervision. In this paper, we propose BLIP, a new VLP framework which transfers flexibly to both vision-language understanding and generation tasks. BLIP effectively utilizes the noisy web data by bootstrapping the captions, where a captioner generates synthetic captions and a filter removes the noisy ones. We achieve state-of-the-art results on a wide range of vision-language tasks, such as image-text retrieval (+2.7% in average recall@1), image captioning (+2.8% in CIDEr), and VQA (+1.6% in VQA score). BLIP also demonstrates strong generalization ability when directly transferred to videolanguage tasks in a zero-shot manner. Code, models, and datasets are released.* -This model was contributed by [INSERT YOUR HF USERNAME HERE](https://huggingface.co/). -The original code can be found [here](). +This model was contributed by [ybelkada](https://huggingface.co/ybelkada). +The original code can be found [here](https://github.com/salesforce/BLIP). ## BlipConfig From 31e4339dfcbb49e9f0343f85d5a3b72673606642 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Sun, 11 Dec 2022 13:35:27 +0000 Subject: [PATCH 18/96] remove from tokenization auto --- src/transformers/models/auto/tokenization_auto.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index 0489750c605e..e2fa1ddd7ce6 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -77,13 +77,6 @@ ("biogpt", ("BioGptTokenizer", None)), ("blenderbot", ("BlenderbotTokenizer", "BlenderbotTokenizerFast")), ("blenderbot-small", ("BlenderbotSmallTokenizer", None)), - ( - "blip", - ( - "CLIPTokenizer", - "CLIPTokenizerFast" if is_tokenizers_available() else None, - ), - ), ("bloom", (None, "BloomTokenizerFast" if is_tokenizers_available() else None)), ("byt5", ("ByT5Tokenizer", None)), ( From da7f972f82f3e0c284877b36beb699112a49d12f Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Sun, 11 Dec 2022 13:53:41 +0000 Subject: [PATCH 19/96] fix config --- .../models/blip/configuration_blip.py | 105 +++++++++++------- .../models/blip/modeling_blip_text.py | 10 +- 2 files changed, 72 insertions(+), 43 deletions(-) diff --git a/src/transformers/models/blip/configuration_blip.py b/src/transformers/models/blip/configuration_blip.py index 637073e87b1b..0feb0530dcad 100644 --- a/src/transformers/models/blip/configuration_blip.py +++ b/src/transformers/models/blip/configuration_blip.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" BLIP model configuration""" +""" Blip model configuration""" import copy import os @@ -25,28 +25,49 @@ logger = logging.get_logger(__name__) BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP = { - "ybelkada/blip-base": "https://huggingface.co/ybelkada/blip-base/resolve/main/config.json", + "Salesforce/blip-vqa-base": "https://huggingface.co/Salesforce/blip-vqa-base/resolve/main/config.json", + "Salesforce/blip-vqa-base-capfit": ( + "https://huggingface.co/Salesforce/blip-vqa-base-capfit/resolve/main/config.json" + ), + "Salesforce/blip-image-captioning-base": ( + "https://huggingface.co/Salesforce/blip-image-captioning-base/resolve/main/config.json" + ), + "Salesforce/blip-image-captioning-large": ( + "https://huggingface.co/Salesforce/blip-image-captioning-large/resolve/main/config.json" + ), + "Salesforce/blip-retrieval-coco-base": ( + "https://huggingface.co/Salesforce/blip-retrieval-coco-base/resolve/main/config.json" + ), + "Salesforce/blip-retrieval-coco-large": ( + "https://huggingface.co/Salesforce/blip-retrieval-coco-large/resolve/main/config.json" + ), + "Salesforce/blip-retrieval-flikr-base": ( + "https://huggingface.co/Salesforce/blip-retrieval-flikr-base/resolve/main/config.json" + ), + "Salesforce/blip-retrieval-flikr-large": ( + "https://huggingface.co/Salesforce/blip-retrieval-flikr-large/resolve/main/config.json" + ), } class BlipTextConfig(PretrainedConfig): r""" - This is the configuration class to store the configuration of a [`BLIPModel`]. It is used to instantiate an BLIP - model according to the specified arguments, defining the model architecture. Instantiating a configuration with the - defaults will yield a similar configuration to that of the BLIP - [ybelkada/blip-base](https://huggingface.co/ybelkada/blip-base) architecture. + This is the configuration class to store the configuration of a [`BlipTextModel`]. It is used to instantiate an + `BlipText` model according to the specified arguments, defining the model architecture. Instantiating a + configuration with the defaults will yield a similar configuration to that of the `BlipText` used by the [base + architectures](https://huggingface.co/Salesforce/blip-vqa-base). Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PretrainedConfig`] for more information. Args: - vocab_size (`int`, *optional*, defaults to 49408): - Vocabulary size of the BLIP text model. Defines the number of different tokens that can be represented by - the `inputs_ids` passed when calling [`BLIPModel`]. - hidden_size (`int`, *optional*, defaults to 512): + vocab_size (`int`, *optional*, defaults to 30522): + Vocabulary size of the `Blip` text model. Defines the number of different tokens that can be represented by + the `inputs_ids` passed when calling [`BlipModel`]. + hidden_size (`int`, *optional*, defaults to 768): Dimensionality of the encoder layers and the pooler layer. - intermediate_size (`int`, *optional*, defaults to 2048): + intermediate_size (`int`, *optional*, defaults to 3072): Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. num_hidden_layers (`int`, *optional*, defaults to 12): Number of hidden layers in the Transformer encoder. @@ -68,17 +89,29 @@ class BlipTextConfig(PretrainedConfig): initializer_factor (`float``, *optional*, defaults to 1): A factor for initializing all weight matrices (should be kept to 1, used internally for initialization testing). + bos_token_id (`int`, *optional*, defaults to 30522): + The id of the `beginning-of-sequence` token. + eos_token_id (`int`, *optional*, defaults to 2): + The id of the `end-of-sequence` token. + pad_token_id (`int`, *optional*, defaults to 0): + The id of the `padding` token. + sep_token_id (`int`, *optional*, defaults to 102): + The id of the `separator` token. + is_decoder (`bool`, *optional*, defaults to `False`): + Whether the model is used as a decoder. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). Example: ```python - >>> from transformers import BlipTextConfig, BLIPTextModel + >>> from transformers import BlipTextConfig, BlipTextModel - >>> # Initializing a BlipTextConfig with ybelkada/blip-base style configuration + >>> # Initializing a BlipTextConfig with Salesforce/blip-vqa-base style configuration >>> configuration = BlipTextConfig() - >>> # Initializing a BLIPTextModel (with random weights) from the ybelkada/blip-base style configuration - >>> model = BLIPTextModel(configuration) + >>> # Initializing a BlipTextModel (with random weights) from the Salesforce/blip-vqa-base style configuration + >>> model = BlipTextModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config @@ -100,14 +133,11 @@ def __init__( attention_probs_dropout_prob=0.0, initializer_range=0.02, initializer_factor=1.0, - type_vocab_size=3072, pad_token_id=0, bos_token_id=30522, eos_token_id=2, - is_decoder=True, - add_cross_attention=True, - use_token_type_embed=False, sep_token_id=102, + is_decoder=True, use_cache=True, **kwargs ): @@ -120,7 +150,6 @@ def __init__( ) self.vocab_size = vocab_size - self.type_vocab_size = type_vocab_size self.hidden_size = hidden_size self.intermediate_size = intermediate_size self.projection_dim = projection_dim @@ -133,9 +162,7 @@ def __init__( self.initializer_range = initializer_range self.initializer_factor = initializer_factor self.attention_probs_dropout_prob = attention_probs_dropout_prob - self.add_cross_attention = add_cross_attention self.is_decoder = is_decoder - self.use_token_type_embed = use_token_type_embed self.use_cache = use_cache @classmethod @@ -158,10 +185,10 @@ def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], class BlipVisionConfig(PretrainedConfig): r""" - This is the configuration class to store the configuration of a [`BLIPModel`]. It is used to instantiate an BLIP - model according to the specified arguments, defining the model architecture. Instantiating a configuration with the - defaults will yield a similar configuration to that of the BLIP - [ybelkada/blip-base](https://huggingface.co/ybelkada/blip-base) architecture. + This is the configuration class to store the configuration of a [`BlipVisionModel`]. It is used to instantiate a + Blip vision model according to the specified arguments, defining the model architecture. Instantiating a + configuration defaults will yield a similar configuration to that of the Blip-base + [Salesforce/blip-vqa-base](https://huggingface.co/Salesforce/blip-vqa-base) architecture. Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PretrainedConfig`] for more information. @@ -197,13 +224,13 @@ class BlipVisionConfig(PretrainedConfig): Example: ```python - >>> from transformers import BlipVisionConfig, BLIPVisionModel + >>> from transformers import BlipVisionConfig, BlipVisionModel - >>> # Initializing a BlipVisionConfig with ybelkada/blip-base style configuration + >>> # Initializing a BlipVisionConfig with Salesforce/blip-vqa-base style configuration >>> configuration = BlipVisionConfig() - >>> # Initializing a BLIPVisionModel (with random weights) from the ybelkada/blip-base style configuration - >>> model = BLIPVisionModel(configuration) + >>> # Initializing a BlipVisionModel (with random weights) from the Salesforce/blip-vqa-base style configuration + >>> model = BlipVisionModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config @@ -266,10 +293,10 @@ def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], class BlipConfig(PretrainedConfig): r""" - [`BlipConfig`] is the configuration class to store the configuration of a [`BLIPModel`]. It is used to instantiate - BLIP model according to the specified arguments, defining the text model and vision model configs. Instantiating a - configuration with the defaults will yield a similar configuration to that of the BLIP - [ybelkada/blip-base](https://huggingface.co/ybelkada/blip-base) architecture. + [`BlipConfig`] is the configuration class to store the configuration of a [`BlipModel`]. It is used to instantiate + Blip model according to the specified arguments, defining the text model and vision model configs. Instantiating a + configuration with the defaults will yield a similar configuration to that of the Blip-base + [Salesforce/blip-vqa-base](https://huggingface.co/Salesforce/blip-vqa-base) architecture. Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the documentation from [`PretrainedConfig`] for more information. @@ -283,19 +310,21 @@ class BlipConfig(PretrainedConfig): Dimentionality of text and vision projection layers. logit_scale_init_value (`float`, *optional*, defaults to 2.6592): The inital value of the *logit_scale* paramter. Default is used as per the original BLIP implementation. + image_text_hidden_size (`int`, *optional*, defaults to 768): + Dimentionality of the hidden state of the image-text fusion layer. kwargs (*optional*): Dictionary of keyword arguments. Example: ```python - >>> from transformers import BlipConfig, BLIPModel + >>> from transformers import BlipConfig, BlipModel - >>> # Initializing a BlipConfig with ybelkada/blip-base style configuration + >>> # Initializing a BlipConfig with Salesforce/blip-vqa-base style configuration >>> configuration = BlipConfig() - >>> # Initializing a BLIPModel (with random weights) from the ybelkada/blip-base style configuration - >>> model = BLIPModel(configuration) + >>> # Initializing a BlipPModel (with random weights) from the Salesforce/blip-vqa-base style configuration + >>> model = BlipModel(configuration) >>> # Accessing the model configuration >>> configuration = model.config diff --git a/src/transformers/models/blip/modeling_blip_text.py b/src/transformers/models/blip/modeling_blip_text.py index 6660090fd34d..d17ecad55d84 100644 --- a/src/transformers/models/blip/modeling_blip_text.py +++ b/src/transformers/models/blip/modeling_blip_text.py @@ -311,8 +311,8 @@ def __init__(self, config, layer_num): self.seq_len_dim = 1 self.attention = BlipTextAttention(config) self.layer_num = layer_num - if self.config.add_cross_attention: - self.crossattention = BlipTextAttention(config, is_cross_attention=self.config.add_cross_attention) + if self.config.is_decoder: + self.crossattention = BlipTextAttention(config, is_cross_attention=self.config.is_decoder) self.intermediate = BlipTextIntermediate(config) self.output = BlipTextOutput(config) @@ -390,7 +390,7 @@ def forward( ): all_hidden_states = () if output_hidden_states else None all_self_attentions = () if output_attentions else None - all_cross_attentions = () if output_attentions and self.config.add_cross_attention else None + all_cross_attentions = () if output_attentions and self.config.is_decoder else None next_decoder_cache = () if use_cache else None @@ -557,8 +557,8 @@ class BlipTextModel(BlipTextPreTrainedModel): The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of cross-attention is added between the self-attention layers, following the architecture described in [Attention is all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit, - Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. argument and `add_cross_attention` set to `True`; - an `encoder_hidden_states` is then expected as an input to the forward pass. + Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin. argument and `is_decoder` set to `True`; an + `encoder_hidden_states` is then expected as an input to the forward pass. """ def __init__(self, config, add_pooling_layer=True): From 2f3b6dd1ac195e4ffb9da9b9bed84946acd6394e Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Sun, 11 Dec 2022 13:55:02 +0000 Subject: [PATCH 20/96] change order --- src/transformers/models/blip/configuration_blip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/blip/configuration_blip.py b/src/transformers/models/blip/configuration_blip.py index 0feb0530dcad..4ff09b0dbbf5 100644 --- a/src/transformers/models/blip/configuration_blip.py +++ b/src/transformers/models/blip/configuration_blip.py @@ -133,9 +133,9 @@ def __init__( attention_probs_dropout_prob=0.0, initializer_range=0.02, initializer_factor=1.0, - pad_token_id=0, bos_token_id=30522, eos_token_id=2, + pad_token_id=0, sep_token_id=102, is_decoder=True, use_cache=True, From 5a1fd7a46ddab41ee3cfcd0928be77242f439817 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Sun, 11 Dec 2022 14:07:01 +0000 Subject: [PATCH 21/96] add `# Copied from` --- src/transformers/models/blip/feature_extraction_blip.py | 6 ++++++ src/transformers/models/blip/processing_blip.py | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/blip/feature_extraction_blip.py b/src/transformers/models/blip/feature_extraction_blip.py index 155d22e1b461..ccabc2006dc2 100644 --- a/src/transformers/models/blip/feature_extraction_blip.py +++ b/src/transformers/models/blip/feature_extraction_blip.py @@ -46,6 +46,7 @@ logger = logging.get_logger(__name__) +# Copied from transformers.models.vilt.image_processing_vilt.max_across_indices def max_across_indices(values: Iterable[Any]) -> List[Any]: """ Return the maximum value across all indices of an iterable of values. @@ -53,6 +54,7 @@ def max_across_indices(values: Iterable[Any]) -> List[Any]: return [max(values_i) for values_i in zip(*values)] +# Copied from transformers.models.vilt.image_processing_vilt.pad def pad( image: np.ndarray, output_size: Tuple[int, int], @@ -93,6 +95,7 @@ def pad( return padded_image +# Copied from transformers.models.vilt.image_processing_vilt.make_pixel_mask def make_pixel_mask(image: np.ndarray, output_size: Tuple[int, int]) -> np.ndarray: """ Make a pixel mask for the image, where 1 indicates a valid pixel and 0 indicates padding. @@ -109,6 +112,7 @@ def make_pixel_mask(image: np.ndarray, output_size: Tuple[int, int]) -> np.ndarr return mask +# Copied from transformers.models.vilt.image_processing_vilt.get_max_dimensions def get_max_dimensions(images: List[np.ndarray]) -> List[int]: """ Get the maximum height and width across all images in a batch. @@ -124,6 +128,7 @@ def get_max_dimensions(images: List[np.ndarray]) -> List[int]: return (max_height, max_width) +# Copied from transformers.models.vilt.image_processing_vilt.get_resize_output_image_size def get_resize_output_image_size( input_image: np.ndarray, shorter: int = 800, longer: int = 1333, size_divisor: int = 32 ) -> Tuple[int, int]: @@ -151,6 +156,7 @@ def get_resize_output_image_size( return new_height, new_width +# Copied from transformers.models.vilt.image_processing_vilt.ViltImageProcessor with ViltImageProcessor->BlipFeatureExtractor and ViLT->Blip class BlipFeatureExtractor(BaseImageProcessor): r""" Constructs a ViLT image processor. diff --git a/src/transformers/models/blip/processing_blip.py b/src/transformers/models/blip/processing_blip.py index 04972ec9661f..cb905702f7c9 100644 --- a/src/transformers/models/blip/processing_blip.py +++ b/src/transformers/models/blip/processing_blip.py @@ -23,9 +23,10 @@ from ...utils import TensorType +# Copied from transformers.models.vilt.processing_vilt.ViltProcessor with Vilt->Blip and ViLT->Blip class BlipProcessor(ProcessorMixin): r""" - Constructs a Blip processor which wraps a BERT tokenizer and Blip feature extractor into a single processor. + Constructs a ViLT processor which wraps a BERT tokenizer and ViLT feature extractor into a single processor. [`BlipProcessor`] offers all the functionalities of [`BlipFeatureExtractor`] and [`BertTokenizerFast`]. See the docstring of [`~BlipProcessor.__call__`] and [`~BlipProcessor.decode`] for more information. From 6387aecd03e498ef58dcf5da17509531171ee9e9 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Sun, 11 Dec 2022 14:23:39 +0000 Subject: [PATCH 22/96] few fixes - add correct license on modeling text - remove dummy argument --- src/transformers/models/blip/modeling_blip.py | 68 ++++++------------- .../models/blip/modeling_blip_text.py | 38 ++++++++--- 2 files changed, 48 insertions(+), 58 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 3df85c73489c..a5a14774422f 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -1,5 +1,5 @@ # coding=utf-8 -# Copyright 2022 The OpenAI Team Authors and The HuggingFace Team. All rights reserved. +# Copyright 2022 The Salesforce Team Authors and The HuggingFace Team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -37,10 +37,17 @@ logger = logging.get_logger(__name__) -_CHECKPOINT_FOR_DOC = "ybelkada/blip-base" +_CHECKPOINT_FOR_DOC = "Salesforce/blip-vqa-base" BLIP_PRETRAINED_MODEL_ARCHIVE_LIST = [ - "ybelkada/blip-base", + "Salesforce/blip-vqa-base", + "Salesforce/blip-vqa-base-capfit", + "Salesforce/blip-image-captioning-base", + "Salesforce/blip-image-captioning-large", + "Salesforce/blip-retrieval-coco-base", + "Salesforce/blip-retrieval-coco-large", + "Salesforce/blip-retrieval-flikr-base", + "Salesforce/blip-retrieval-flikr-large", # See all BLIP models at https://huggingface.co/models?filter=blip ] @@ -90,7 +97,7 @@ class BlipForConditionalGenerationModelOutput(ModelOutput): @dataclass -class BlipVisionModelOutput(ModelOutput): +class BlipTextVisionModelOutput(ModelOutput): """ Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states. @@ -119,35 +126,6 @@ class BlipVisionModelOutput(ModelOutput): attentions: Optional[Tuple[torch.FloatTensor]] = None -@dataclass -class BlipTextModelOutput(ModelOutput): - """ - Base class for text model's outputs that also contains a pooling of the last hidden states. - - Args: - text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`): - The text embeddings obtained by applying the projection layer to the pooler_output. - last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): - Sequence of hidden-states at the output of the last layer of the model. - hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): - Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + - one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. - - Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. - attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): - Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, - sequence_length)`. - - Attentions weights after the attention softmax, used to compute the weighted average in the self-attention - heads. - """ - - text_embeds: Optional[torch.FloatTensor] = None - last_hidden_state: torch.FloatTensor = None - hidden_states: Optional[Tuple[torch.FloatTensor]] = None - attentions: Optional[Tuple[torch.FloatTensor]] = None - - @dataclass class BlipOutput(ModelOutput): """ @@ -581,7 +559,6 @@ def forward( self, inputs_embeds, attention_mask: Optional[torch.Tensor] = None, - causal_attention_mask: Optional[torch.Tensor] = None, output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, @@ -598,13 +575,6 @@ def forward( - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) - causal_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): - Causal mask for the text model. Mask values selected in `[0, 1]`: - - - 1 for tokens that are **not masked**, - - 0 for tokens that are **masked**. - [What are attention masks?](../glossary#attention-mask) output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under @@ -980,7 +950,7 @@ def get_input_embeddings(self) -> nn.Module: return self.vision_model.embeddings.patch_embedding @add_start_docstrings_to_model_forward(BLIP_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BlipVisionModelOutput, config_class=BlipVisionConfig) + @replace_return_docstrings(output_type=BlipTextVisionModelOutput, config_class=BlipVisionConfig) def forward( self, input_ids: torch.LongTensor, @@ -989,7 +959,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, BlipVisionModelOutput]: + ) -> Union[Tuple, BlipTextVisionModelOutput]: r""" Returns: @@ -1131,7 +1101,7 @@ def get_input_embeddings(self) -> nn.Module: return self.vision_model.embeddings.patch_embedding @add_start_docstrings_to_model_forward(BLIP_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BlipVisionModelOutput, config_class=BlipVisionConfig) + @replace_return_docstrings(output_type=BlipTextVisionModelOutput, config_class=BlipVisionConfig) def forward( self, input_ids: torch.LongTensor, @@ -1142,7 +1112,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, BlipVisionModelOutput]: + ) -> Union[Tuple, BlipTextVisionModelOutput]: r""" Returns: @@ -1207,7 +1177,7 @@ def forward( outputs = (decoder_loss, image_embeds, vision_outputs[0]) + vision_outputs[2:] return tuple(output for output in outputs if output is not None) - return BlipVisionModelOutput( + return BlipTextVisionModelOutput( loss=decoder_loss, image_embeds=image_embeds, last_hidden_state=vision_outputs.last_hidden_state, @@ -1323,7 +1293,7 @@ def get_input_embeddings(self) -> nn.Module: return self.vision_model.embeddings.patch_embedding @add_start_docstrings_to_model_forward(BLIP_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BlipVisionModelOutput, config_class=BlipVisionConfig) + @replace_return_docstrings(output_type=BlipTextVisionModelOutput, config_class=BlipVisionConfig) def forward( self, input_ids: torch.LongTensor, @@ -1333,7 +1303,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, BlipVisionModelOutput]: + ) -> Union[Tuple, BlipTextVisionModelOutput]: r""" Returns: @@ -1397,7 +1367,7 @@ def forward( outputs = (output, vision_outputs[0]) + vision_outputs[2:] return tuple(output for output in outputs if output is not None) - return BlipVisionModelOutput( + return BlipTextVisionModelOutput( image_embeds=output, last_hidden_state=vision_outputs.last_hidden_state, hidden_states=vision_outputs.hidden_states, diff --git a/src/transformers/models/blip/modeling_blip_text.py b/src/transformers/models/blip/modeling_blip_text.py index d17ecad55d84..341091031413 100644 --- a/src/transformers/models/blip/modeling_blip_text.py +++ b/src/transformers/models/blip/modeling_blip_text.py @@ -1,12 +1,18 @@ -""" - * Copyright (c) 2022, salesforce.com, inc. - * All rights reserved. - * SPDX-License-Identifier: BSD-3-Clause - * For full license text, see LICENSE.txt file in the repo root or https://opensource.org/licenses/BSD-3-Clause - * By Junnan Li - * Based on huggingface code base - * https://github.com/huggingface/transformers/blob/v4.15.0/src/transformers/models/bert -""" +# coding=utf-8 +# Copyright 2022 The Salesforce Team Authors and The HuggingFace Team. All rights reserved. +# +# Licensed under the BSD-3-clause license (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://opensource.org/licenses/BSD-3-Clause +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import math from typing import Tuple @@ -36,6 +42,7 @@ logger = logging.get_logger(__name__) +# Copied from https://github.com/salesforce/BLIP/blob/main/models/med.py#L52 class BlipTextEmbeddings(nn.Module): """Construct the embeddings from word and position embeddings.""" @@ -80,6 +87,7 @@ def forward(self, input_ids=None, position_ids=None, inputs_embeds=None, past_ke return embeddings +# Copied from https://github.com/salesforce/BLIP/blob/main/models/med.py#L97 class BlipTextSelfAttention(nn.Module): def __init__(self, config, is_cross_attention): super().__init__() @@ -211,6 +219,7 @@ def forward( return outputs +# Copied from https://github.com/salesforce/BLIP/blob/main/models/med.py#228 class BlipTextSelfOutput(nn.Module): def __init__(self, config): super().__init__() @@ -225,6 +234,7 @@ def forward(self, hidden_states, input_tensor): return hidden_states +# Copied from https://github.com/salesforce/BLIP/blob/main/models/med.py#242 class BlipTextAttention(nn.Module): def __init__(self, config, is_cross_attention=False): super().__init__() @@ -274,6 +284,7 @@ def forward( return outputs +# Copied from https://github.com/salesforce/BLIP/blob/main/models/med.py#L291 class BlipTextIntermediate(nn.Module): def __init__(self, config): super().__init__() @@ -289,6 +300,7 @@ def forward(self, hidden_states): return hidden_states +# Copied from https://github.com/salesforce/BLIP/blob/main/models/med.py#L306 class BlipTextOutput(nn.Module): def __init__(self, config): super().__init__() @@ -367,6 +379,7 @@ def feed_forward_chunk(self, attention_output): return layer_output +# Copied from https://github.com/salesforce/BLIP/blob/main/models/med.py#L386 class BlipTextEncoder(nn.Module): def __init__(self, config): super().__init__() @@ -467,6 +480,7 @@ def custom_forward(*inputs): ) +# Copied from https://github.com/salesforce/BLIP/blob/3a29b7410476bf5f2ba0955827390eb6ea1f4f9d/models/med.py#L486 class BlipTextPooler(nn.Module): def __init__(self, config): super().__init__() @@ -482,6 +496,7 @@ def forward(self, hidden_states): return pooled_output +# Copied from https://github.com/salesforce/BLIP/blob/3a29b7410476bf5f2ba0955827390eb6ea1f4f9d/models/med.py#L501 class BlipTextPredictionHeadTransform(nn.Module): def __init__(self, config): super().__init__() @@ -499,6 +514,7 @@ def forward(self, hidden_states): return hidden_states +# Copied from https://github.com/salesforce/BLIP/blob/3a29b7410476bf5f2ba0955827390eb6ea1f4f9d/models/med.py#L518 class BlipTextLMPredictionHead(nn.Module): def __init__(self, config): super().__init__() @@ -519,6 +535,7 @@ def forward(self, hidden_states): return hidden_states +# Copied from https://github.com/salesforce/BLIP/blob/main/models/med.py#L538 class BlipTextOnlyMLMHead(nn.Module): def __init__(self, config): super().__init__() @@ -529,6 +546,7 @@ def forward(self, sequence_output): return prediction_scores +# Copied from https://github.com/salesforce/BLIP/blob/main/models/med.py#L548 class BlipTextPreTrainedModel(PreTrainedModel): """ An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained @@ -552,6 +570,7 @@ def _init_weights(self, module): module.bias.data.zero_() +# Copied from https://github.com/salesforce/BLIP/blob/3a29b7410476bf5f2ba0955827390eb6ea1f4f9d/models/med.py#L571 class BlipTextModel(BlipTextPreTrainedModel): """ The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of @@ -796,6 +815,7 @@ def forward( ) +# Copied from https://github.com/salesforce/BLIP/blob/main/models/med.py#L811 class BlipTextLMHeadModel(BlipTextPreTrainedModel): _keys_to_ignore_on_load_unexpected = [r"pooler"] From fef5a21edafcbc0605a6f66c8ca1afdde1228635 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Mon, 12 Dec 2022 15:06:55 +0100 Subject: [PATCH 23/96] Apply suggestions from code review Co-authored-by: NielsRogge <48327001+NielsRogge@users.noreply.github.com> --- docs/source/en/model_doc/blip.mdx | 4 ++-- src/transformers/models/blip/modeling_blip.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/en/model_doc/blip.mdx b/docs/source/en/model_doc/blip.mdx index d8a2d9a53d3f..f1be5278f380 100644 --- a/docs/source/en/model_doc/blip.mdx +++ b/docs/source/en/model_doc/blip.mdx @@ -74,9 +74,9 @@ The original code can be found [here](https://github.com/salesforce/BLIP). - forward -## BlipForVisualQuestionAnswering +## BlipForQuestionAnswering -[[autodoc]] BlipForVisualQuestionAnswering +[[autodoc]] BlipForQuestionAnswering - forward diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index a5a14774422f..fca4e542ce6e 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -1078,7 +1078,7 @@ def generate( """, BLIP_START_DOCSTRING, ) -class BlipForVisualQuestionAnswering(BlipPreTrainedModel): +class BlipForQuestionAnswering(BlipPreTrainedModel): config_class = BlipConfig _keys_to_ignore_on_load_missing = [r"text_decoder.cls.predictions.decoder.bias"] @@ -1121,7 +1121,7 @@ def forward( ```python >>> from PIL import Image >>> import requests - >>> from transformers import CLIPProcessor, BLIPForImageCaptioning + >>> from transformers import BLIPProcessor, BLIPForImageCaptioning >>> model = BLIPForImageCaptioning.from_pretrained("ybelkada/blip-base") >>> processor = CLIPProcessor.from_pretrained("ybelkada/blip-base") From fb399147cc36b4f148417fae421c63a22d3e66d9 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Mon, 12 Dec 2022 14:08:35 +0000 Subject: [PATCH 24/96] replace name --- src/transformers/__init__.py | 4 ++-- src/transformers/models/blip/__init__.py | 4 ++-- src/transformers/utils/dummy_pt_objects.py | 2 +- tests/models/blip/test_modeling_blip.py | 4 ++-- utils/check_repo.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 36fe06b93db6..3eb467e48fe7 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -1099,7 +1099,7 @@ "BLIP_PRETRAINED_MODEL_ARCHIVE_LIST", "BlipForConditionalGeneration", "BlipForImageTextRetrieval", - "BlipForVisualQuestionAnswering", + "BlipForQuestionAnswering", "BlipModel", "BlipPreTrainedModel", "BlipTextModel", @@ -4262,7 +4262,7 @@ BLIP_PRETRAINED_MODEL_ARCHIVE_LIST, BlipForConditionalGeneration, BlipForImageTextRetrieval, - BlipForVisualQuestionAnswering, + BlipForQuestionAnswering, BlipModel, BlipPreTrainedModel, BlipTextModel, diff --git a/src/transformers/models/blip/__init__.py b/src/transformers/models/blip/__init__.py index f9fae1a5c070..47f25b65749c 100644 --- a/src/transformers/models/blip/__init__.py +++ b/src/transformers/models/blip/__init__.py @@ -50,7 +50,7 @@ "BlipModel", "BlipPreTrainedModel", "BlipForConditionalGeneration", - "BlipForVisualQuestionAnswering", + "BlipForQuestionAnswering", "BlipVisionModel", "BlipTextModel", "BlipForImageTextRetrieval", @@ -78,7 +78,7 @@ BLIP_PRETRAINED_MODEL_ARCHIVE_LIST, BlipForConditionalGeneration, BlipForImageTextRetrieval, - BlipForVisualQuestionAnswering, + BlipForQuestionAnswering, BlipModel, BlipPreTrainedModel, BlipTextModel, diff --git a/src/transformers/utils/dummy_pt_objects.py b/src/transformers/utils/dummy_pt_objects.py index cd1c612bb889..968f1c84f5cd 100644 --- a/src/transformers/utils/dummy_pt_objects.py +++ b/src/transformers/utils/dummy_pt_objects.py @@ -1129,7 +1129,7 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["torch"]) -class BlipForVisualQuestionAnswering(metaclass=DummyObject): +class BlipForQuestionAnswering(metaclass=DummyObject): _backends = ["torch"] def __init__(self, *args, **kwargs): diff --git a/tests/models/blip/test_modeling_blip.py b/tests/models/blip/test_modeling_blip.py index dbb3530a0fd5..6ca9b70b3d29 100644 --- a/tests/models/blip/test_modeling_blip.py +++ b/tests/models/blip/test_modeling_blip.py @@ -44,7 +44,7 @@ from transformers import ( BlipForConditionalGeneration, BlipForImageTextRetrieval, - BlipForVisualQuestionAnswering, + BlipForQuestionAnswering, BlipModel, BlipTextModel, BlipVisionModel, @@ -574,7 +574,7 @@ class BlipTextImageModelTest(ModelTesterMixin, unittest.TestCase): all_model_classes = ( ( BlipForConditionalGeneration, - BlipForVisualQuestionAnswering, + BlipForQuestionAnswering, BlipForImageTextRetrieval, ) if is_torch_available() diff --git a/utils/check_repo.py b/utils/check_repo.py index 0f61b5fbf719..28b55bd8e73e 100644 --- a/utils/check_repo.py +++ b/utils/check_repo.py @@ -149,7 +149,7 @@ # models to ignore for model xxx mapping "BlipForConditionalGeneration", "BlipForImageTextRetrieval", - "BlipForVisualQuestionAnswering", + "BlipForQuestionAnswering", "BlipVisionModel", "BlipTextLMHeadModel", "BlipTextModel", From 051618374e121e05e95a180df4f9ac6dbe4c685b Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Mon, 12 Dec 2022 14:15:49 +0000 Subject: [PATCH 25/96] refactor a bit --- .../models/blip/modeling_blip_text.py | 33 ++++++++----------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip_text.py b/src/transformers/models/blip/modeling_blip_text.py index 341091031413..ad78dd656fb4 100644 --- a/src/transformers/models/blip/modeling_blip_text.py +++ b/src/transformers/models/blip/modeling_blip_text.py @@ -42,7 +42,7 @@ logger = logging.get_logger(__name__) -# Copied from https://github.com/salesforce/BLIP/blob/main/models/med.py#L52 +# Adapted from https://github.com/salesforce/BLIP/blob/main/models/med.py#L52 class BlipTextEmbeddings(nn.Module): """Construct the embeddings from word and position embeddings.""" @@ -87,7 +87,7 @@ def forward(self, input_ids=None, position_ids=None, inputs_embeds=None, past_ke return embeddings -# Copied from https://github.com/salesforce/BLIP/blob/main/models/med.py#L97 +# Adapted from https://github.com/salesforce/BLIP/blob/main/models/med.py#L97 class BlipTextSelfAttention(nn.Module): def __init__(self, config, is_cross_attention): super().__init__() @@ -115,7 +115,6 @@ def __init__(self, config, is_cross_attention): if self.position_embedding_type == "relative_key" or self.position_embedding_type == "relative_key_query": self.max_position_embeddings = config.max_position_embeddings self.distance_embedding = nn.Embedding(2 * config.max_position_embeddings - 1, self.attention_head_size) - self.save_attention = False def save_attn_gradients(self, attn_gradients): self.attn_gradients = attn_gradients @@ -195,10 +194,6 @@ def forward( # Normalize the attention scores to probabilities. attention_probs = nn.Softmax(dim=-1)(attention_scores) - if is_cross_attention and self.save_attention: - self.save_attention_map(attention_probs) - attention_probs.register_hook(self.save_attn_gradients) - # This is actually dropping out entire tokens to attend to, which might # seem a bit unusual, but is taken from the original Transformer paper. attention_probs_dropped = self.dropout(attention_probs) @@ -219,7 +214,7 @@ def forward( return outputs -# Copied from https://github.com/salesforce/BLIP/blob/main/models/med.py#228 +# Adapted from https://github.com/salesforce/BLIP/blob/main/models/med.py#228 class BlipTextSelfOutput(nn.Module): def __init__(self, config): super().__init__() @@ -234,7 +229,7 @@ def forward(self, hidden_states, input_tensor): return hidden_states -# Copied from https://github.com/salesforce/BLIP/blob/main/models/med.py#242 +# Adapted from https://github.com/salesforce/BLIP/blob/main/models/med.py#242 class BlipTextAttention(nn.Module): def __init__(self, config, is_cross_attention=False): super().__init__() @@ -284,7 +279,7 @@ def forward( return outputs -# Copied from https://github.com/salesforce/BLIP/blob/main/models/med.py#L291 +# Adapted from https://github.com/salesforce/BLIP/blob/main/models/med.py#L291 class BlipTextIntermediate(nn.Module): def __init__(self, config): super().__init__() @@ -300,7 +295,7 @@ def forward(self, hidden_states): return hidden_states -# Copied from https://github.com/salesforce/BLIP/blob/main/models/med.py#L306 +# Adapted from https://github.com/salesforce/BLIP/blob/main/models/med.py#L306 class BlipTextOutput(nn.Module): def __init__(self, config): super().__init__() @@ -379,7 +374,7 @@ def feed_forward_chunk(self, attention_output): return layer_output -# Copied from https://github.com/salesforce/BLIP/blob/main/models/med.py#L386 +# Adapted from https://github.com/salesforce/BLIP/blob/main/models/med.py#L386 class BlipTextEncoder(nn.Module): def __init__(self, config): super().__init__() @@ -480,7 +475,7 @@ def custom_forward(*inputs): ) -# Copied from https://github.com/salesforce/BLIP/blob/3a29b7410476bf5f2ba0955827390eb6ea1f4f9d/models/med.py#L486 +# Adapted from https://github.com/salesforce/BLIP/blob/3a29b7410476bf5f2ba0955827390eb6ea1f4f9d/models/med.py#L486 class BlipTextPooler(nn.Module): def __init__(self, config): super().__init__() @@ -496,7 +491,7 @@ def forward(self, hidden_states): return pooled_output -# Copied from https://github.com/salesforce/BLIP/blob/3a29b7410476bf5f2ba0955827390eb6ea1f4f9d/models/med.py#L501 +# Adapted from https://github.com/salesforce/BLIP/blob/3a29b7410476bf5f2ba0955827390eb6ea1f4f9d/models/med.py#L501 class BlipTextPredictionHeadTransform(nn.Module): def __init__(self, config): super().__init__() @@ -514,7 +509,7 @@ def forward(self, hidden_states): return hidden_states -# Copied from https://github.com/salesforce/BLIP/blob/3a29b7410476bf5f2ba0955827390eb6ea1f4f9d/models/med.py#L518 +# Adapted from https://github.com/salesforce/BLIP/blob/3a29b7410476bf5f2ba0955827390eb6ea1f4f9d/models/med.py#L518 class BlipTextLMPredictionHead(nn.Module): def __init__(self, config): super().__init__() @@ -535,7 +530,7 @@ def forward(self, hidden_states): return hidden_states -# Copied from https://github.com/salesforce/BLIP/blob/main/models/med.py#L538 +# Adapted from https://github.com/salesforce/BLIP/blob/main/models/med.py#L538 class BlipTextOnlyMLMHead(nn.Module): def __init__(self, config): super().__init__() @@ -546,7 +541,7 @@ def forward(self, sequence_output): return prediction_scores -# Copied from https://github.com/salesforce/BLIP/blob/main/models/med.py#L548 +# Adapted from https://github.com/salesforce/BLIP/blob/main/models/med.py#L548 class BlipTextPreTrainedModel(PreTrainedModel): """ An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained @@ -570,7 +565,7 @@ def _init_weights(self, module): module.bias.data.zero_() -# Copied from https://github.com/salesforce/BLIP/blob/3a29b7410476bf5f2ba0955827390eb6ea1f4f9d/models/med.py#L571 +# Adapted from https://github.com/salesforce/BLIP/blob/3a29b7410476bf5f2ba0955827390eb6ea1f4f9d/models/med.py#L571 class BlipTextModel(BlipTextPreTrainedModel): """ The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of @@ -815,7 +810,7 @@ def forward( ) -# Copied from https://github.com/salesforce/BLIP/blob/main/models/med.py#L811 +# Adapted from https://github.com/salesforce/BLIP/blob/main/models/med.py#L811 class BlipTextLMHeadModel(BlipTextPreTrainedModel): _keys_to_ignore_on_load_unexpected = [r"pooler"] From 41663020d2097e299742a756d9da054cb2beae65 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Mon, 12 Dec 2022 14:19:12 +0000 Subject: [PATCH 26/96] more refactor --- .../models/blip/modeling_blip_text.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip_text.py b/src/transformers/models/blip/modeling_blip_text.py index ad78dd656fb4..8e1837c106a1 100644 --- a/src/transformers/models/blip/modeling_blip_text.py +++ b/src/transformers/models/blip/modeling_blip_text.py @@ -22,6 +22,8 @@ from torch import Tensor, device, nn from torch.nn import CrossEntropyLoss +from typing import List, Optional, Tuple, Union + from transformers.activations import ACT2FN from transformers.modeling_outputs import ( BaseModelOutputWithPastAndCrossAttentions, @@ -215,6 +217,7 @@ def forward( # Adapted from https://github.com/salesforce/BLIP/blob/main/models/med.py#228 +# Copied from transformers.models.bert.modeling_bert.BertSelfOutput with Bert -> BlipText class BlipTextSelfOutput(nn.Module): def __init__(self, config): super().__init__() @@ -222,7 +225,7 @@ def __init__(self, config): self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) self.dropout = nn.Dropout(config.hidden_dropout_prob) - def forward(self, hidden_states, input_tensor): + def forward(self, hidden_states: torch.Tensor, input_tensor: torch.Tensor) -> torch.Tensor: hidden_states = self.dense(hidden_states) hidden_states = self.dropout(hidden_states) hidden_states = self.LayerNorm(hidden_states + input_tensor) @@ -257,13 +260,13 @@ def prune_heads(self, heads): def forward( self, - hidden_states, - attention_mask=None, - head_mask=None, - encoder_hidden_states=None, - encoder_attention_mask=None, - past_key_value=None, - output_attentions=False, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.FloatTensor] = None, + head_mask: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + past_key_value: Optional[Tuple[Tuple[torch.FloatTensor]]] = None, + output_attentions: Optional[bool] = False, ): self_outputs = self.self( hidden_states, From d5073eee9a93a5a14b62cc84d04f29f3ad6104b6 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Mon, 12 Dec 2022 14:25:47 +0000 Subject: [PATCH 27/96] remove unused arg --- .../models/blip/modeling_blip_text.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip_text.py b/src/transformers/models/blip/modeling_blip_text.py index 8e1837c106a1..bf912c7eb2e4 100644 --- a/src/transformers/models/blip/modeling_blip_text.py +++ b/src/transformers/models/blip/modeling_blip_text.py @@ -282,7 +282,7 @@ def forward( return outputs -# Adapted from https://github.com/salesforce/BLIP/blob/main/models/med.py#L291 +# Copied from transformers.models.bert.modeling_bert.BertIntermediate with Bert -> BlipText class BlipTextIntermediate(nn.Module): def __init__(self, config): super().__init__() @@ -292,13 +292,13 @@ def __init__(self, config): else: self.intermediate_act_fn = config.hidden_act - def forward(self, hidden_states): + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: hidden_states = self.dense(hidden_states) hidden_states = self.intermediate_act_fn(hidden_states) return hidden_states -# Adapted from https://github.com/salesforce/BLIP/blob/main/models/med.py#L306 +# Copied from transformers.models.bert.modeling_bert.BertOutput with Bert -> BlipText class BlipTextOutput(nn.Module): def __init__(self, config): super().__init__() @@ -306,7 +306,7 @@ def __init__(self, config): self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) self.dropout = nn.Dropout(config.hidden_dropout_prob) - def forward(self, hidden_states, input_tensor): + def forward(self, hidden_states: torch.Tensor, input_tensor: torch.Tensor) -> torch.Tensor: hidden_states = self.dense(hidden_states) hidden_states = self.dropout(hidden_states) hidden_states = self.LayerNorm(hidden_states + input_tensor) @@ -335,7 +335,6 @@ def forward( encoder_attention_mask=None, past_key_value=None, output_attentions=False, - mode=None, ): # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None @@ -397,7 +396,6 @@ def forward( output_attentions=False, output_hidden_states=False, return_dict=True, - mode="multimodal", ): all_hidden_states = () if output_hidden_states else None all_self_attentions = () if output_attentions else None @@ -434,7 +432,6 @@ def custom_forward(*inputs): layer_head_mask, encoder_hidden_states, encoder_attention_mask, - mode=mode, ) else: layer_outputs = layer_module( @@ -445,7 +442,6 @@ def custom_forward(*inputs): encoder_attention_mask, past_key_value, output_attentions, - mode=mode, ) hidden_states = layer_outputs[0] @@ -685,7 +681,6 @@ def forward( output_hidden_states=None, return_dict=None, is_decoder=False, - mode="multimodal", ): r""" encoder_hidden_states (: @@ -795,7 +790,6 @@ def forward( output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict, - mode=mode, ) sequence_output = encoder_outputs[0] pooled_output = self.pooler(sequence_output) if self.pooler is not None else None @@ -849,7 +843,6 @@ def forward( return_logits=False, is_decoder=True, reduction="mean", - mode="multimodal", ): r""" encoder_hidden_states (: @@ -897,7 +890,6 @@ def forward( output_hidden_states=output_hidden_states, return_dict=return_dict, is_decoder=is_decoder, - mode=mode, ) sequence_output = outputs[0] From 4dee8fe6d3accde8f3a1aad0d8bd5b375b66e2aa Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Mon, 12 Dec 2022 14:30:34 +0000 Subject: [PATCH 28/96] make fixup + remove some `# Adapted from ...` --- .../models/blip/modeling_blip_text.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip_text.py b/src/transformers/models/blip/modeling_blip_text.py index bf912c7eb2e4..8bb778ab9b90 100644 --- a/src/transformers/models/blip/modeling_blip_text.py +++ b/src/transformers/models/blip/modeling_blip_text.py @@ -15,15 +15,13 @@ import math -from typing import Tuple +from typing import Optional, Tuple import torch import torch.utils.checkpoint from torch import Tensor, device, nn from torch.nn import CrossEntropyLoss -from typing import List, Optional, Tuple, Union - from transformers.activations import ACT2FN from transformers.modeling_outputs import ( BaseModelOutputWithPastAndCrossAttentions, @@ -216,7 +214,6 @@ def forward( return outputs -# Adapted from https://github.com/salesforce/BLIP/blob/main/models/med.py#228 # Copied from transformers.models.bert.modeling_bert.BertSelfOutput with Bert -> BlipText class BlipTextSelfOutput(nn.Module): def __init__(self, config): @@ -474,14 +471,14 @@ def custom_forward(*inputs): ) -# Adapted from https://github.com/salesforce/BLIP/blob/3a29b7410476bf5f2ba0955827390eb6ea1f4f9d/models/med.py#L486 +# Copied from transformers.models.bert.modeling_bert.BertPooler with Bert->BlipText class BlipTextPooler(nn.Module): def __init__(self, config): super().__init__() self.dense = nn.Linear(config.hidden_size, config.hidden_size) self.activation = nn.Tanh() - def forward(self, hidden_states): + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: # We "pool" the model by simply taking the hidden state corresponding # to the first token. first_token_tensor = hidden_states[:, 0] @@ -490,7 +487,7 @@ def forward(self, hidden_states): return pooled_output -# Adapted from https://github.com/salesforce/BLIP/blob/3a29b7410476bf5f2ba0955827390eb6ea1f4f9d/models/med.py#L501 +# Copied from transformers.models.bert.modeling_bert.BertPredictionHeadTransform with Bert->BlipText class BlipTextPredictionHeadTransform(nn.Module): def __init__(self, config): super().__init__() @@ -501,14 +498,14 @@ def __init__(self, config): self.transform_act_fn = config.hidden_act self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) - def forward(self, hidden_states): + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: hidden_states = self.dense(hidden_states) hidden_states = self.transform_act_fn(hidden_states) hidden_states = self.LayerNorm(hidden_states) return hidden_states -# Adapted from https://github.com/salesforce/BLIP/blob/3a29b7410476bf5f2ba0955827390eb6ea1f4f9d/models/med.py#L518 +# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->BlipText class BlipTextLMPredictionHead(nn.Module): def __init__(self, config): super().__init__() @@ -529,13 +526,13 @@ def forward(self, hidden_states): return hidden_states -# Adapted from https://github.com/salesforce/BLIP/blob/main/models/med.py#L538 +# Copied from transformers.models.bert.modeling_bert.BertOnlyMLMHead with Bert->BlipText class BlipTextOnlyMLMHead(nn.Module): def __init__(self, config): super().__init__() self.predictions = BlipTextLMPredictionHead(config) - def forward(self, sequence_output): + def forward(self, sequence_output: torch.Tensor) -> torch.Tensor: prediction_scores = self.predictions(sequence_output) return prediction_scores From ad08646180453de70021bfd6ea268243d7fa7ec8 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Mon, 12 Dec 2022 15:32:08 +0100 Subject: [PATCH 29/96] Apply suggestions from code review Co-authored-by: NielsRogge <48327001+NielsRogge@users.noreply.github.com> --- src/transformers/models/blip/modeling_blip_text.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip_text.py b/src/transformers/models/blip/modeling_blip_text.py index 8bb778ab9b90..c914be18501b 100644 --- a/src/transformers/models/blip/modeling_blip_text.py +++ b/src/transformers/models/blip/modeling_blip_text.py @@ -576,9 +576,7 @@ def __init__(self, config, add_pooling_layer=True): self.config = config self.embeddings = BlipTextEmbeddings(config) - self.encoder = BlipTextEncoder(config) - self.pooler = BlipTextPooler(config) if add_pooling_layer else None self.post_init() From 194dfa5db4ebc44b28f84734b9f2b66da887cf66 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Mon, 12 Dec 2022 14:37:01 +0000 Subject: [PATCH 30/96] more `# Copied from` --- src/transformers/models/blip/modeling_blip_text.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/models/blip/modeling_blip_text.py b/src/transformers/models/blip/modeling_blip_text.py index c914be18501b..c0146a631ac4 100644 --- a/src/transformers/models/blip/modeling_blip_text.py +++ b/src/transformers/models/blip/modeling_blip_text.py @@ -587,6 +587,7 @@ def get_input_embeddings(self): def set_input_embeddings(self, value): self.embeddings.word_embeddings = value + # Copied from transformers.models.bert.modeling_bert.BertModel._prune_heads def _prune_heads(self, heads_to_prune): """ Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base From 4c9ee7f9153183e085103182d71add15305792c2 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Mon, 12 Dec 2022 15:38:09 +0100 Subject: [PATCH 31/96] Apply suggestions from code review Co-authored-by: NielsRogge <48327001+NielsRogge@users.noreply.github.com> --- src/transformers/models/blip/processing_blip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/blip/processing_blip.py b/src/transformers/models/blip/processing_blip.py index cb905702f7c9..73edd58bf4ac 100644 --- a/src/transformers/models/blip/processing_blip.py +++ b/src/transformers/models/blip/processing_blip.py @@ -26,7 +26,7 @@ # Copied from transformers.models.vilt.processing_vilt.ViltProcessor with Vilt->Blip and ViLT->Blip class BlipProcessor(ProcessorMixin): r""" - Constructs a ViLT processor which wraps a BERT tokenizer and ViLT feature extractor into a single processor. + Constructs a BLIP processor which wraps a BERT tokenizer and BLIP image processor into a single processor. [`BlipProcessor`] offers all the functionalities of [`BlipFeatureExtractor`] and [`BertTokenizerFast`]. See the docstring of [`~BlipProcessor.__call__`] and [`~BlipProcessor.decode`] for more information. From 96536b36b491b39f20e33282669fa83373c2a601 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Mon, 12 Dec 2022 14:49:44 +0000 Subject: [PATCH 32/96] now `generate` supports no prefix --- src/transformers/models/blip/modeling_blip.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index fca4e542ce6e..58d692d85357 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -953,8 +953,8 @@ def get_input_embeddings(self) -> nn.Module: @replace_return_docstrings(output_type=BlipTextVisionModelOutput, config_class=BlipVisionConfig) def forward( self, - input_ids: torch.LongTensor, pixel_values: torch.FloatTensor, + input_ids: Optional[torch.LongTensor] = None, attention_mask: Optional[torch.LongTensor] = None, output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, @@ -981,6 +981,7 @@ def forward( >>> outputs = model(**inputs) >>> image_embeds = outputs.image_embeds ```""" + batch_size, _ = pixel_values.shape[:2] return_dict = return_dict if return_dict is not None else self.config.use_return_dict vision_outputs = self.vision_model( @@ -992,6 +993,9 @@ def forward( image_embeds = vision_outputs[0] + if input_ids is None: + input_ids = torch.LongTensor([[self.decoder_input_ids] * batch_size]).to(image_embeds.device) + decoder_targets = input_ids.masked_fill(input_ids == self.decoder_input_ids, -100) outputs = self.text_decoder( @@ -1017,7 +1021,7 @@ def forward( @torch.no_grad() def generate( - self, input_ids: torch.LongTensor, pixel_values: torch.FloatTensor, **generate_kwargs + self, pixel_values: torch.FloatTensor, input_ids: Optional[torch.LongTensor] = None, **generate_kwargs ) -> torch.LongTensor: r""" Overrides `generate` function to be able to use the model as a conditional generator @@ -1047,6 +1051,7 @@ def generate( >>> image_embeds = outputs.image_embeds ``` """ + batch_size, _ = pixel_values.shape[:2] vision_outputs = self.vision_model( pixel_values=pixel_values, ) @@ -1058,6 +1063,10 @@ def generate( if isinstance(input_ids, list): input_ids = torch.LongTensor(input_ids) + elif input_ids is None: + input_ids = torch.LongTensor( + [[self.decoder_input_ids, self.config.text_config.eos_token_id] * batch_size] + ).to(image_embeds.device) input_ids[:, 0] = self.config.text_config.bos_token_id @@ -1354,7 +1363,6 @@ def forward( input_ids=input_ids, attention_mask=attention_mask, return_dict=return_dict, - mode="text", ) question_embeds = question_embeds[0] if not return_dict else question_embeds.last_hidden_state From af5848ca9eb9df1f5fedc7dd2d4ce97548057221 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Mon, 12 Dec 2022 14:54:42 +0000 Subject: [PATCH 33/96] remove `FeatureExtractor` --- docs/source/en/model_doc/blip.mdx | 4 ++-- src/transformers/__init__.py | 4 ++-- src/transformers/models/blip/__init__.py | 4 ++-- ...re_extraction_blip.py => image_processing_blip.py} | 4 ++-- src/transformers/models/blip/processing_blip.py | 11 +++++------ src/transformers/utils/dummy_vision_objects.py | 2 +- 6 files changed, 14 insertions(+), 15 deletions(-) rename src/transformers/models/blip/{feature_extraction_blip.py => image_processing_blip.py} (99%) diff --git a/docs/source/en/model_doc/blip.mdx b/docs/source/en/model_doc/blip.mdx index f1be5278f380..447835860ece 100644 --- a/docs/source/en/model_doc/blip.mdx +++ b/docs/source/en/model_doc/blip.mdx @@ -85,7 +85,7 @@ The original code can be found [here](https://github.com/salesforce/BLIP). [[autodoc]] BlipProcessor -## BlipFeatureExtractor +## BlipImageProcessor -[[autodoc]] BlipFeatureExtractor +[[autodoc]] BlipImageProcessor - preprocess \ No newline at end of file diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 3eb467e48fe7..0891a229181d 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -755,7 +755,7 @@ _import_structure["image_utils"] = ["ImageFeatureExtractionMixin"] _import_structure["models.beit"].extend(["BeitFeatureExtractor", "BeitImageProcessor"]) _import_structure["models.bit"].extend(["BitImageProcessor"]) - _import_structure["models.blip"].extend(["BlipFeatureExtractor", "BlipProcessor"]) + _import_structure["models.blip"].extend(["BlipImageProcessor", "BlipProcessor"]) _import_structure["models.chinese_clip"].extend(["ChineseCLIPFeatureExtractor", "ChineseCLIPImageProcessor"]) _import_structure["models.clip"].extend(["CLIPFeatureExtractor", "CLIPImageProcessor"]) _import_structure["models.conditional_detr"].extend( @@ -3971,7 +3971,7 @@ from .image_utils import ImageFeatureExtractionMixin from .models.beit import BeitFeatureExtractor, BeitImageProcessor from .models.bit import BitImageProcessor - from .models.blip import BlipFeatureExtractor, BlipProcessor + from .models.blip import BlipImageProcessor, BlipProcessor from .models.chinese_clip import ChineseCLIPFeatureExtractor, ChineseCLIPImageProcessor from .models.clip import CLIPFeatureExtractor, CLIPImageProcessor from .models.conditional_detr import ConditionalDetrFeatureExtractor, ConditionalDetrImageProcessor diff --git a/src/transformers/models/blip/__init__.py b/src/transformers/models/blip/__init__.py index 47f25b65749c..a065457305f8 100644 --- a/src/transformers/models/blip/__init__.py +++ b/src/transformers/models/blip/__init__.py @@ -35,7 +35,7 @@ except OptionalDependencyNotAvailable: pass else: - _import_structure["feature_extraction_blip"] = ["BlipFeatureExtractor"] + _import_structure["feature_extraction_blip"] = ["BlipImageProcessor"] _import_structure["processing_blip"] = ["BlipProcessor"] @@ -65,7 +65,7 @@ except OptionalDependencyNotAvailable: pass else: - from .feature_extraction_blip import BlipFeatureExtractor + from .image_processing_blip import BlipImageProcessor from .processing_blip import BlipProcessor try: diff --git a/src/transformers/models/blip/feature_extraction_blip.py b/src/transformers/models/blip/image_processing_blip.py similarity index 99% rename from src/transformers/models/blip/feature_extraction_blip.py rename to src/transformers/models/blip/image_processing_blip.py index ccabc2006dc2..324f43c6eb94 100644 --- a/src/transformers/models/blip/feature_extraction_blip.py +++ b/src/transformers/models/blip/image_processing_blip.py @@ -156,8 +156,8 @@ def get_resize_output_image_size( return new_height, new_width -# Copied from transformers.models.vilt.image_processing_vilt.ViltImageProcessor with ViltImageProcessor->BlipFeatureExtractor and ViLT->Blip -class BlipFeatureExtractor(BaseImageProcessor): +# Copied from transformers.models.vilt.image_processing_vilt.ViltImageProcessor with Vilt->Blip and ViLT->Blip +class BlipImageProcessor(BaseImageProcessor): r""" Constructs a ViLT image processor. diff --git a/src/transformers/models/blip/processing_blip.py b/src/transformers/models/blip/processing_blip.py index 73edd58bf4ac..dbdbaeac1564 100644 --- a/src/transformers/models/blip/processing_blip.py +++ b/src/transformers/models/blip/processing_blip.py @@ -23,21 +23,20 @@ from ...utils import TensorType -# Copied from transformers.models.vilt.processing_vilt.ViltProcessor with Vilt->Blip and ViLT->Blip class BlipProcessor(ProcessorMixin): r""" Constructs a BLIP processor which wraps a BERT tokenizer and BLIP image processor into a single processor. - [`BlipProcessor`] offers all the functionalities of [`BlipFeatureExtractor`] and [`BertTokenizerFast`]. See the + [`BlipProcessor`] offers all the functionalities of [`BlipImageProcessor`] and [`BertTokenizerFast`]. See the docstring of [`~BlipProcessor.__call__`] and [`~BlipProcessor.decode`] for more information. Args: - feature_extractor (`BlipFeatureExtractor`): - An instance of [`BlipFeatureExtractor`]. The feature extractor is a required input. + feature_extractor (`BlipImageProcessor`): + An instance of [`BlipImageProcessor`]. The feature extractor is a required input. tokenizer (`BertTokenizerFast`): An instance of ['BertTokenizerFast`]. The tokenizer is a required input. """ - feature_extractor_class = "BlipFeatureExtractor" + feature_extractor_class = "BlipImageProcessor" tokenizer_class = ("BertTokenizer", "BertTokenizerFast") def __init__(self, feature_extractor, tokenizer): @@ -65,7 +64,7 @@ def __call__( **kwargs ) -> BatchEncoding: """ - This method uses [`BlipFeatureExtractor.__call__`] method to prepare image(s) for the model, and + This method uses [`BlipImageProcessor.__call__`] method to prepare image(s) for the model, and [`BertTokenizerFast.__call__`] to prepare text for the model. Please refer to the docstring of the above two methods for more information. diff --git a/src/transformers/utils/dummy_vision_objects.py b/src/transformers/utils/dummy_vision_objects.py index ce445b380326..bf04ca14dc59 100644 --- a/src/transformers/utils/dummy_vision_objects.py +++ b/src/transformers/utils/dummy_vision_objects.py @@ -50,7 +50,7 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["vision"]) -class BlipFeatureExtractor(metaclass=DummyObject): +class BlipImageProcessor(metaclass=DummyObject): _backends = ["vision"] def __init__(self, *args, **kwargs): From 62230fc49744255a28db3717144d9cec2abe959d Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Mon, 12 Dec 2022 14:58:07 +0000 Subject: [PATCH 34/96] fix path --- src/transformers/models/blip/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/blip/__init__.py b/src/transformers/models/blip/__init__.py index a065457305f8..563d1977854f 100644 --- a/src/transformers/models/blip/__init__.py +++ b/src/transformers/models/blip/__init__.py @@ -35,7 +35,7 @@ except OptionalDependencyNotAvailable: pass else: - _import_structure["feature_extraction_blip"] = ["BlipImageProcessor"] + _import_structure["image_processing_blip"] = ["BlipImageProcessor"] _import_structure["processing_blip"] = ["BlipProcessor"] From bf089c2f05267558025d197aeca9fe4c0457a429 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Mon, 12 Dec 2022 14:59:45 +0000 Subject: [PATCH 35/96] correct dependency --- src/transformers/models/blip/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/blip/__init__.py b/src/transformers/models/blip/__init__.py index 563d1977854f..9b021adf5b1e 100644 --- a/src/transformers/models/blip/__init__.py +++ b/src/transformers/models/blip/__init__.py @@ -27,6 +27,7 @@ "BlipTextConfig", "BlipVisionConfig", ], + "processing_blip": ["BlipProcessor"], } try: @@ -36,7 +37,6 @@ pass else: _import_structure["image_processing_blip"] = ["BlipImageProcessor"] - _import_structure["processing_blip"] = ["BlipProcessor"] try: @@ -58,6 +58,7 @@ if TYPE_CHECKING: from .configuration_blip import BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP, BlipConfig, BlipTextConfig, BlipVisionConfig + from .processing_blip import BlipProcessor try: if not is_vision_available(): @@ -66,7 +67,6 @@ pass else: from .image_processing_blip import BlipImageProcessor - from .processing_blip import BlipProcessor try: if not is_torch_available(): From 3fc2776cc42a6aff465c0123129b0522111a08e0 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Mon, 12 Dec 2022 15:59:02 +0000 Subject: [PATCH 36/96] fix tests --- .../models/blip/image_processing_blip.py | 18 ++--- src/transformers/models/blip/modeling_blip.py | 49 +++++++----- .../models/blip/processing_blip.py | 44 ++++++----- tests/models/blip/test_modeling_blip.py | 75 +++++++++++++------ 4 files changed, 111 insertions(+), 75 deletions(-) diff --git a/src/transformers/models/blip/image_processing_blip.py b/src/transformers/models/blip/image_processing_blip.py index 324f43c6eb94..e19b84cb290c 100644 --- a/src/transformers/models/blip/image_processing_blip.py +++ b/src/transformers/models/blip/image_processing_blip.py @@ -156,7 +156,6 @@ def get_resize_output_image_size( return new_height, new_width -# Copied from transformers.models.vilt.image_processing_vilt.ViltImageProcessor with Vilt->Blip and ViLT->Blip class BlipImageProcessor(BaseImageProcessor): r""" Constructs a ViLT image processor. @@ -217,8 +216,8 @@ def __init__( do_pad = kwargs.pop("pad_and_return_pixel_mask") super().__init__(**kwargs) - size = size if size is not None else {"shortest_edge": 384} - size = get_size_dict(size, default_to_square=False) + size = size if size is not None else 384 + size = get_size_dict(size, default_to_square=True) self.do_resize = do_resize self.size = size @@ -235,7 +234,6 @@ def resize( self, image: np.ndarray, size: Dict[str, int], - size_divisor: int = 32, resample: PILImageResampling = PILImageResampling.BICUBIC, data_format: Optional[Union[str, ChannelDimension]] = None, **kwargs @@ -259,12 +257,8 @@ def resize( data_format (`str` or `ChannelDimension`, *optional*): The channel dimension format of the image. If not provided, it will be the same as the input image. """ - size = get_size_dict(size, default_to_square=False) - if "shortest_edge" not in size: - raise ValueError(f"The `size` dictionary must contain the key `shortest_edge`. Got {size.keys()}") - shorter = size["shortest_edge"] - longer = int(1333 / 800 * shorter) - output_size = get_resize_output_image_size(image, shorter=shorter, longer=longer, size_divisor=size_divisor) + size = get_size_dict(size, default_to_square=True) + output_size = (size["width"], size["height"]) return resize(image, size=output_size, resample=resample, data_format=data_format, **kwargs) def rescale( @@ -474,9 +468,7 @@ def preprocess( images = [to_numpy_array(image) for image in images] if do_resize: - images = [ - self.resize(image=image, size=size, size_divisor=size_divisor, resample=resample) for image in images - ] + images = [self.resize(image=image, size=size, resample=resample) for image in images] if do_rescale: images = [self.rescale(image=image, scale=rescale_factor) for image in images] diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 58d692d85357..0811fc96b597 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -192,6 +192,7 @@ def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: return embeddings +# Copied from transformers.models.clip.modeling_clip.CLIPTextEmbeddings with CLIP->Blip class BlipTextEmbeddings(nn.Module): def __init__(self, config: BlipTextConfig): super().__init__() @@ -314,6 +315,7 @@ def forward( return outputs +# Copied from transformers.models.clip.modeling_clip.CLIPMLP with CLIP->Blip class BlipMLP(nn.Module): def __init__(self, config): super().__init__() @@ -754,8 +756,8 @@ def get_text_features( ```python >>> from transformers import CLIPTokenizer, BlipModel - >>> model = BlipModel.from_pretrained("ybelkada/blip-base") - >>> tokenizer = CLIPTokenizer.from_pretrained("ybelkada/blip-base") + >>> model = BlipModel.from_pretrained("Salesforce/blip-image-captioning-base") + >>> tokenizer = CLIPTokenizer.from_pretrained("Salesforce/blip-image-captioning-base") >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") >>> text_features = model.get_text_features(**inputs) @@ -801,8 +803,8 @@ def get_image_features( >>> import requests >>> from transformers import CLIPProcessor, BlipModel - >>> model = BlipModel.from_pretrained("ybelkada/blip-base") - >>> processor = CLIPProcessor.from_pretrained("ybelkada/blip-base") + >>> model = BlipModel.from_pretrained("Salesforce/blip-image-captioning-base") + >>> processor = CLIPProcessor.from_pretrained("Salesforce/blip-image-captioning-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -853,8 +855,8 @@ def forward( >>> import requests >>> from transformers import CLIPProcessor, BlipModel - >>> model = BlipModel.from_pretrained("ybelkada/blip-base") - >>> processor = CLIPProcessor.from_pretrained("ybelkada/blip-base") + >>> model = BlipModel.from_pretrained("Salesforce/blip-image-captioning-base") + >>> processor = CLIPProcessor.from_pretrained("Salesforce/blip-image-captioning-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -933,6 +935,7 @@ def forward( class BlipForConditionalGeneration(BlipPreTrainedModel): config_class = BlipConfig _keys_to_ignore_on_load_missing = [r"text_decoder.cls.predictions.decoder.bias"] + main_input_name = "pixel_values" def __init__(self, config: BlipConfig): super().__init__(config) @@ -970,8 +973,8 @@ def forward( >>> import requests >>> from transformers import CLIPProcessor, BLIPForImageCaptioning - >>> model = BLIPForImageCaptioning.from_pretrained("ybelkada/blip-base") - >>> processor = CLIPProcessor.from_pretrained("ybelkada/blip-base") + >>> model = BLIPForImageCaptioning.from_pretrained("Salesforce/blip-image-captioning-base") + >>> processor = CLIPProcessor.from_pretrained("Salesforce/blip-image-captioning-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -1021,7 +1024,13 @@ def forward( @torch.no_grad() def generate( - self, pixel_values: torch.FloatTensor, input_ids: Optional[torch.LongTensor] = None, **generate_kwargs + self, + pixel_values: torch.FloatTensor, + input_ids: Optional[torch.LongTensor] = None, + pixel_mask: Optional[torch.LongTensor] = None, + token_type_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.LongTensor] = None, + **generate_kwargs ) -> torch.LongTensor: r""" Overrides `generate` function to be able to use the model as a conditional generator @@ -1039,8 +1048,8 @@ def generate( >>> import requests >>> from transformers import BlipTokenizer, BlipForImageCaptioning - >>> model = BLIPForImageCaptioning.from_pretrained("ybelkada/blip-base") - >>> processor = CLIPProcessor.from_pretrained("ybelkada/blip-base") + >>> model = BLIPForImageCaptioning.from_pretrained("Salesforce/blip-image-captioning-base") + >>> processor = CLIPProcessor.from_pretrained("Salesforce/blip-image-captioning-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -1059,7 +1068,6 @@ def generate( image_embeds = vision_outputs[0] image_atts = torch.ones(image_embeds.size()[:-1], dtype=torch.long).to(image_embeds.device) - model_kwargs = {"encoder_hidden_states": image_embeds, "encoder_attention_mask": image_atts} if isinstance(input_ids, list): input_ids = torch.LongTensor(input_ids) @@ -1069,13 +1077,16 @@ def generate( ).to(image_embeds.device) input_ids[:, 0] = self.config.text_config.bos_token_id + attention_mask = attention_mask[:, :-1] if attention_mask is not None else None outputs = self.text_decoder.generate( input_ids=input_ids[:, :-1], eos_token_id=self.config.text_config.sep_token_id, pad_token_id=self.config.text_config.pad_token_id, + attention_mask=attention_mask, + encoder_hidden_states=image_embeds, + encoder_attention_mask=image_atts, **generate_kwargs, - **model_kwargs, ) return outputs @@ -1132,8 +1143,8 @@ def forward( >>> import requests >>> from transformers import BLIPProcessor, BLIPForImageCaptioning - >>> model = BLIPForImageCaptioning.from_pretrained("ybelkada/blip-base") - >>> processor = CLIPProcessor.from_pretrained("ybelkada/blip-base") + >>> model = BLIPForImageCaptioning.from_pretrained("Salesforce/blip-image-captioning-base") + >>> processor = CLIPProcessor.from_pretrained("Salesforce/blip-image-captioning-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -1219,8 +1230,8 @@ def generate( >>> import requests >>> from transformers import BlipTokenizer, BlipForImageCaptioning - >>> model = BLIPForImageCaptioning.from_pretrained("ybelkada/blip-base") - >>> processor = CLIPProcessor.from_pretrained("ybelkada/blip-base") + >>> model = BLIPForImageCaptioning.from_pretrained("Salesforce/blip-image-captioning-base") + >>> processor = CLIPProcessor.from_pretrained("Salesforce/blip-image-captioning-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -1323,8 +1334,8 @@ def forward( >>> import requests >>> from transformers import CLIPProcessor, BLIPForImageCaptioning - >>> model = BLIPForImageCaptioning.from_pretrained("ybelkada/blip-base") - >>> processor = CLIPProcessor.from_pretrained("ybelkada/blip-base") + >>> model = BLIPForImageCaptioning.from_pretrained("Salesforce/blip-image-captioning-base") + >>> processor = CLIPProcessor.from_pretrained("Salesforce/blip-image-captioning-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) diff --git a/src/transformers/models/blip/processing_blip.py b/src/transformers/models/blip/processing_blip.py index dbdbaeac1564..c11b41efa0e1 100644 --- a/src/transformers/models/blip/processing_blip.py +++ b/src/transformers/models/blip/processing_blip.py @@ -69,29 +69,33 @@ def __call__( Please refer to the docstring of the above two methods for more information. """ - encoding = self.tokenizer( - text=text, - add_special_tokens=add_special_tokens, - padding=padding, - truncation=truncation, - max_length=max_length, - stride=stride, - pad_to_multiple_of=pad_to_multiple_of, - return_token_type_ids=return_token_type_ids, - return_attention_mask=return_attention_mask, - return_overflowing_tokens=return_overflowing_tokens, - return_special_tokens_mask=return_special_tokens_mask, - return_offsets_mapping=return_offsets_mapping, - return_length=return_length, - verbose=verbose, - return_tensors=return_tensors, - **kwargs, - ) + if text is not None: + text_encoding = self.tokenizer( + text=text, + add_special_tokens=add_special_tokens, + padding=padding, + truncation=truncation, + max_length=max_length, + stride=stride, + pad_to_multiple_of=pad_to_multiple_of, + return_token_type_ids=return_token_type_ids, + return_attention_mask=return_attention_mask, + return_overflowing_tokens=return_overflowing_tokens, + return_special_tokens_mask=return_special_tokens_mask, + return_offsets_mapping=return_offsets_mapping, + return_length=return_length, + verbose=verbose, + return_tensors=return_tensors, + **kwargs, + ) + else: + text_encoding = None # add pixel_values + pixel_mask encoding_feature_extractor = self.feature_extractor(images, return_tensors=return_tensors) - encoding.update(encoding_feature_extractor) + if text_encoding is not None: + encoding_feature_extractor.update(text_encoding) - return encoding + return encoding_feature_extractor def batch_decode(self, *args, **kwargs): """ diff --git a/tests/models/blip/test_modeling_blip.py b/tests/models/blip/test_modeling_blip.py index 6ca9b70b3d29..6f6de9fbd785 100644 --- a/tests/models/blip/test_modeling_blip.py +++ b/tests/models/blip/test_modeling_blip.py @@ -55,7 +55,7 @@ if is_vision_available(): from PIL import Image - from transformers import CLIPProcessor + from transformers import BlipProcessor class BlipVisionModelTester: @@ -609,6 +609,32 @@ def test_retain_grad_hidden_states_attentions(self): def test_model_common_attributes(self): pass + def test_forward_signature(self): + config, _ = self.model_tester.prepare_config_and_inputs_for_common() + + for model_class in self.all_model_classes: + model = model_class(config) + signature = inspect.signature(model.forward) + # signature.parameters is an OrderedDict => so arg_names order is deterministic + arg_names = [*signature.parameters.keys()] + + if model.config.is_encoder_decoder: + expected_arg_names = [ + "input_ids", + "attention_mask", + "decoder_input_ids", + "decoder_attention_mask", + ] + expected_arg_names.extend( + ["head_mask", "decoder_head_mask", "cross_attn_head_mask", "encoder_outputs"] + if "head_mask" and "decoder_head_mask" and "cross_attn_head_mask" in arg_names + else ["encoder_outputs"] + ) + self.assertListEqual(arg_names[: len(expected_arg_names)], expected_arg_names) + else: + expected_arg_names = ["input_ids"] if model_class != BlipForConditionalGeneration else ["pixel_values"] + self.assertListEqual(arg_names[:1], expected_arg_names) + def test_training(self): if not self.model_tester.is_training: return @@ -740,39 +766,42 @@ def test_model_from_pretrained(self): # We will verify our results on an image of cute cats def prepare_img(): - url = "http://images.cocodataset.org/val2017/000000039769.jpg" + url = "https://storage.googleapis.com/sfr-vision-language-research/BLIP/demo.jpg" im = Image.open(requests.get(url, stream=True).raw) return im @require_vision @require_torch +@slow class BlipModelIntegrationTest(unittest.TestCase): - @slow - def test_inference(self): - model_name = "ybelkada/blip-base" - model = BlipModel.from_pretrained(model_name).to(torch_device) - processor = CLIPProcessor.from_pretrained(model_name) - + def test_inference_image_captioning(self): + model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base").to(torch_device) + processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") image = prepare_img() - inputs = processor( - text=["a photo of a cat", "a photo of a dog"], images=image, padding=True, return_tensors="pt" - ).to(torch_device) - # forward pass - with torch.no_grad(): - outputs = model(**inputs) + # image only + inputs = processor(image=image).to(torch_device) - # verify the logits - self.assertEqual( - outputs.logits_per_image.shape, - torch.Size((inputs.pixel_values.shape[0], inputs.input_ids.shape[0])), - ) + predictions = model.generate(**inputs) + + # Test output + self.assertEqual(predictions[0].tolist(), [30522, 1037, 2450, 3564, 2006, 1996, 3509, 2007, 2014, 3899, 102]) + + # image and context + context = ["a picture of"] + inputs = processor(image=image, text=context).to(torch_device) + + predictions = model.generate(**inputs) + + # Test output self.assertEqual( - outputs.logits_per_text.shape, - torch.Size((inputs.input_ids.shape[0], inputs.pixel_values.shape[0])), + predictions[0].tolist(), + [30522, 1037, 3861, 1997, 1037, 2450, 3564, 2006, 1996, 3509, 2007, 2014, 3899, 102], ) - expected_logits = torch.tensor([[24.5701, 19.3049]], device=torch_device) + def test_inference_vqa(self): + pass - self.assertTrue(torch.allclose(outputs.logits_per_image, expected_logits, atol=1e-3)) + def test_inference_itm(self): + pass From 03ed567cf40f888db8d4bbb44bcf4abd98ccce66 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Mon, 12 Dec 2022 17:47:21 +0000 Subject: [PATCH 37/96] few fixes --- src/transformers/models/blip/modeling_blip.py | 197 +++++++++++------- .../models/blip/processing_blip.py | 31 ++- 2 files changed, 153 insertions(+), 75 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 0811fc96b597..796574e0bfec 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -52,8 +52,7 @@ ] -# contrastive loss function, adapted from -# https://sachinruk.github.io/blog/pytorch/pytorch%20lightning/loss%20function/gpu/2021/03/07/BLIP.html +# Copied from transformers.models.clip.modeling_clip.contrastive_loss def contrastive_loss(logits: torch.Tensor) -> torch.Tensor: return nn.functional.cross_entropy(logits, torch.arange(len(logits), device=logits.device)) @@ -68,9 +67,46 @@ def blip_loss(similarity: torch.Tensor) -> torch.Tensor: @dataclass class BlipForConditionalGenerationModelOutput(ModelOutput): """ - Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states. + Adapted from the base class for vision model's outputs that also contains image embeddings of the pooling of the + last hidden states. This class also adds the loss term from the text decoder. Args: + loss (*optional*, *optional* returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`): + Languge modeling loss from the text decoder. + image_embeds (*torch.FloatTensor* of shape *(batch_size, output_dim)* *optional* returned when model is initialized with *with_projection=True*): + The image embeddings obtained by applying the projection layer to the pooler_output. + last_hidden_state (*torch.FloatTensor* of shape *(batch_size, sequence_length, hidden_size)*): + Sequence of hidden-states at the output of the last layer of the model. + hidden_states (*tuple(torch.FloatTensor)*, *optional*, returned when *output_hidden_states=True* is passed or when *config.output_hidden_states=True*): + Tuple of *torch.FloatTensor* (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape *(batch_size, sequence_length, hidden_size)*. + + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. + attentions (*tuple(torch.FloatTensor)*, *optional*, returned when *output_attentions=True* is passed or when *config.output_attentions=True*): + Tuple of *torch.FloatTensor* (one for each layer) of shape *(batch_size, num_heads, sequence_length, + sequence_length)*. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + """ + + loss: Optional[Tuple[torch.FloatTensor]] = None + decoder_logits: Optional[Tuple[torch.FloatTensor]] = None + image_embeds: Optional[torch.FloatTensor] = None + last_hidden_state: torch.FloatTensor = None + hidden_states: Optional[Tuple[torch.FloatTensor]] = None + attentions: Optional[Tuple[torch.FloatTensor]] = None + + +@dataclass +class BlipTextVisionModelOutput(ModelOutput): + """ + Adapted from the base class for vision model's outputs that also contains image embeddings of the pooling of the + last hidden states. This class also adds the loss term from the text decoder. + + Args: + loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): + Languge modeling loss from the text decoder. image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`): The image embeddings obtained by applying the projection layer to the pooler_output. last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): @@ -88,8 +124,7 @@ class BlipForConditionalGenerationModelOutput(ModelOutput): heads. """ - loss: Optional[Tuple[torch.FloatTensor]] = None - decoder_logits: Optional[Tuple[torch.FloatTensor]] = None + loss: Optional[torch.FloatTensor] = None image_embeds: Optional[torch.FloatTensor] = None last_hidden_state: torch.FloatTensor = None hidden_states: Optional[Tuple[torch.FloatTensor]] = None @@ -97,11 +132,17 @@ class BlipForConditionalGenerationModelOutput(ModelOutput): @dataclass -class BlipTextVisionModelOutput(ModelOutput): +class BlipITMModelOutput(ModelOutput): """ - Base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states. + Adapted from the base class for vision model's outputs that also contains image embeddings of the pooling of the + last hidden states. This class also adds the loss term from the text decoder as well as the image-text similarity + scores. Args: + itm_score (`torch.FloatTensor`): + The image-text similarity scores. + loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): + Languge modeling loss from the text decoder. image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`): The image embeddings obtained by applying the projection layer to the pooler_output. last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): @@ -111,19 +152,26 @@ class BlipTextVisionModelOutput(ModelOutput): one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. + vision_pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`, *optional*): + Last layer hidden-state of the vision of the vision-only branch of the model. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + question_embeds (`torch.FloatTensor`): + The question embeddings obtained by the text projection layer. """ + itm_score: Optional[torch.FloatTensor] = None loss: Optional[torch.FloatTensor] = None image_embeds: Optional[torch.FloatTensor] = None last_hidden_state: torch.FloatTensor = None hidden_states: Optional[Tuple[torch.FloatTensor]] = None + vision_pooler_output: Optional[torch.FloatTensor] = None attentions: Optional[Tuple[torch.FloatTensor]] = None + question_embeds: Optional[Tuple[torch.FloatTensor]] = None @dataclass @@ -243,27 +291,11 @@ def __init__(self, config): self.qkv = nn.Linear(self.embed_dim, 3 * self.embed_dim) - self.proj = nn.Linear(self.embed_dim, self.embed_dim) + self.projection = nn.Linear(self.embed_dim, self.embed_dim) def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() - def _split_heads(self, fused_qkv: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - """ - Split the last dimension into (num_heads, head_dim) without making any copies, results share same memory - storage as `fused_qkv` - - Args: - fused_qkv (`torch.tensor`, *required*): [batch_size, seq_length, num_heads * 3 * head_dim] - - Returns: - query: [batch_size, seq_length, num_heads, head_dim] key: [batch_size, seq_length, num_heads, head_dim] - value: [batch_size, seq_length, num_heads, head_dim] - """ - batch_size, seq_length, three_times_hidden_size = fused_qkv.shape - fused_qkv = fused_qkv.view(batch_size, seq_length, self.num_heads, 3, self.head_dim) - return fused_qkv[..., 0, :], fused_qkv[..., 1, :], fused_qkv[..., 2, :] - def forward( self, hidden_states: torch.Tensor, @@ -275,7 +307,6 @@ def forward( bsz, tgt_len, embed_dim = hidden_states.size() mixed_qkv = self.qkv(hidden_states) - # query_states, key_states, value_states = self._split_heads(mixed_qkv) mixed_qkv = ( self.qkv(hidden_states) .reshape(bsz, tgt_len, 3, self.num_heads, embed_dim // self.num_heads) @@ -285,7 +316,7 @@ def forward( mixed_qkv[0], mixed_qkv[1], mixed_qkv[2], - ) # make torchscript happy (cannot use tensor as tuple) + ) # Take the dot product between "query" and "key" to get the raw attention scores. attention_scores = torch.matmul(query_states, key_states.transpose(-1, -2)) @@ -308,7 +339,7 @@ def forward( new_context_layer_shape = context_layer.size()[:-2] + (self.embed_dim,) context_layer = context_layer.reshape(new_context_layer_shape) - output = self.proj(context_layer) + output = self.projection(context_layer) outputs = (output, attention_probs) if output_attentions else (output, None) @@ -406,9 +437,9 @@ def _init_weights(self, module): nn.init.normal_(module.qkv.weight, std=in_proj_std) if module.qkv.bias is not None: nn.init.normal_(module.qkv.bias, std=in_proj_std) - nn.init.normal_(module.proj.weight, std=in_proj_std) - if module.proj.bias is not None: - nn.init.normal_(module.proj.bias, std=in_proj_std) + nn.init.normal_(module.projection.weight, std=in_proj_std) + if module.projection.bias is not None: + nn.init.normal_(module.projection.bias, std=in_proj_std) elif isinstance(module, BlipMLP): factor = self.config.initializer_factor in_proj_std = ( @@ -463,8 +494,7 @@ def _set_gradient_checkpointing(self, module, value=False): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`CLIPTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. + Indices can be obtained using [`BlipProcessor`]. See [`BlipProcessor.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): @@ -493,7 +523,7 @@ def _set_gradient_checkpointing(self, module, value=False): Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`CLIPFeatureExtractor`]. See [`CLIPFeatureExtractor.__call__`] for details. + [`BlipImageProcessor`]. See [`BlipImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. @@ -510,8 +540,7 @@ def _set_gradient_checkpointing(self, module, value=False): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. - Indices can be obtained using [`CLIPTokenizer`]. See [`PreTrainedTokenizer.encode`] and - [`PreTrainedTokenizer.__call__`] for details. + Indices can be obtained using [`BlipProcessor`]. See [`BlipProcessor.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): @@ -528,7 +557,7 @@ def _set_gradient_checkpointing(self, module, value=False): [What are position IDs?](../glossary#position-ids) pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Padding will be ignored by default should you provide it. Pixel values can be obtained using - [`CLIPFeatureExtractor`]. See [`CLIPFeatureExtractor.__call__`] for details. + [`BlipImageProcessor`]. See [`BlipImageProcessor.__call__`] for details. return_loss (`bool`, *optional*): Whether or not to return the contrastive loss. output_attentions (`bool`, *optional*): @@ -754,12 +783,12 @@ def get_text_features( Examples: ```python - >>> from transformers import CLIPTokenizer, BlipModel + >>> from transformers import BlipProcessor, BlipModel >>> model = BlipModel.from_pretrained("Salesforce/blip-image-captioning-base") - >>> tokenizer = CLIPTokenizer.from_pretrained("Salesforce/blip-image-captioning-base") + >>> processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") - >>> inputs = tokenizer(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") + >>> inputs = processor(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") >>> text_features = model.get_text_features(**inputs) ```""" # Use BLIP model's config for some fields (if specified) instead of those of vision & text components. @@ -801,10 +830,10 @@ def get_image_features( ```python >>> from PIL import Image >>> import requests - >>> from transformers import CLIPProcessor, BlipModel + >>> from transformers import BlipProcessor, BlipModel >>> model = BlipModel.from_pretrained("Salesforce/blip-image-captioning-base") - >>> processor = CLIPProcessor.from_pretrained("Salesforce/blip-image-captioning-base") + >>> processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -853,10 +882,10 @@ def forward( ```python >>> from PIL import Image >>> import requests - >>> from transformers import CLIPProcessor, BlipModel + >>> from transformers import BlipProcessor, BlipModel >>> model = BlipModel.from_pretrained("Salesforce/blip-image-captioning-base") - >>> processor = CLIPProcessor.from_pretrained("Salesforce/blip-image-captioning-base") + >>> processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -928,7 +957,9 @@ def forward( @add_start_docstrings( """ - BLIP Vision Model with a projection layer on top (a linear layer on top of the pooled output). + BLIP Model trained to generate captions on images. The model can also take as input a text input. The text decoder + will start generating the caption from the text input. If no text input is provided, the decoder will start with + the [BOS] token only. """, BLIP_START_DOCSTRING, ) @@ -971,10 +1002,10 @@ def forward( ```python >>> from PIL import Image >>> import requests - >>> from transformers import CLIPProcessor, BLIPForImageCaptioning + >>> from transformers import BlipProcessor, BlipForConditionalGeneration - >>> model = BLIPForImageCaptioning.from_pretrained("Salesforce/blip-image-captioning-base") - >>> processor = CLIPProcessor.from_pretrained("Salesforce/blip-image-captioning-base") + >>> model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base") + >>> processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -1033,23 +1064,30 @@ def generate( **generate_kwargs ) -> torch.LongTensor: r""" - Overrides `generate` function to be able to use the model as a conditional generator + Overrides *generate* function to be able to use the model as a conditional generator Parameters: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): - The sequence used as a prompt for the generation. - pixel_values (`torch.FloatTensor` of shape `(batch_size, image_width, image_height)`: + pixel_values (*torch.FloatTensor* of shape *(batch_size, image_width, image_height)*: Input image to be processed + input_ids (*torch.LongTensor* of shape *(batch_size, sequence_length)*, *optional*): + The sequence used as a prompt for the generation. + pixel_mask (*torch.LongTensor* of shape *(batch_size, image_width, image_height)*, *optional*): + Mask to be used for the image - not used but kept for compatibility with the *BlipProcessor* + token_type_ids (*torch.LongTensor* of shape *(batch_size, sequence_length)*, *optional*): + Segment token indices to indicate first and second portions of the inputs. Not used but kept for + compatibility with the *BlipProcessor* + attention_mask (*torch.LongTensor* of shape *(batch_size, sequence_length)*, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: Examples: ```python >>> from PIL import Image >>> import requests - >>> from transformers import BlipTokenizer, BlipForImageCaptioning + >>> from transformers import BlipProcessor, BlipForImageCaptioning - >>> model = BLIPForImageCaptioning.from_pretrained("Salesforce/blip-image-captioning-base") - >>> processor = CLIPProcessor.from_pretrained("Salesforce/blip-image-captioning-base") + >>> model = BlipForImageCaptioning.from_pretrained("Salesforce/blip-image-captioning-base") + >>> processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -1060,6 +1098,7 @@ def generate( >>> image_embeds = outputs.image_embeds ``` """ + batch_size, _ = pixel_values.shape[:2] vision_outputs = self.vision_model( pixel_values=pixel_values, @@ -1094,7 +1133,9 @@ def generate( @add_start_docstrings( """ - BLIP Vision Model with a projection layer on top (a linear layer on top of the pooled output). + BLIP Model fine-tuned for question answering. The model consists of a vision encoder, a text encoder as well as a + text decoder. The vision encoder will encode the input image, the text encoder will encode the input question + together with the encoding of the image, and the text decoder will output the answer to the question. """, BLIP_START_DOCSTRING, ) @@ -1141,18 +1182,17 @@ def forward( ```python >>> from PIL import Image >>> import requests - >>> from transformers import BLIPProcessor, BLIPForImageCaptioning + >>> from transformers import BlipProcessor, BlipForQuestionAnswering - >>> model = BLIPForImageCaptioning.from_pretrained("Salesforce/blip-image-captioning-base") - >>> processor = CLIPProcessor.from_pretrained("Salesforce/blip-image-captioning-base") + >>> model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base") + >>> processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor(images=image, return_tensors="pt") - >>> outputs = model(**inputs) - >>> image_embeds = outputs.image_embeds + >>> outputs = model.generate(**inputs) ```""" return_dict = return_dict if return_dict is not None else self.config.use_return_dict batch_size, _ = input_ids.shape @@ -1215,13 +1255,21 @@ def generate( **generate_kwargs ) -> torch.LongTensor: r""" - Overrides `generate` function to be able to use the model as a conditional generator + Overrides *generate* function to be able to use the model as a conditional generator Parameters: - input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + input_ids (*torch.LongTensor* of shape *(batch_size, sequence_length)*): The sequence used as a prompt for the generation. - pixel_values (`torch.FloatTensor` of shape `(batch_size, image_width, image_height)`: + pixel_values (*torch.FloatTensor* of shape *(batch_size, image_width, image_height)*: Input image to be processed + attention_mask (*torch.LongTensor* of shape *(batch_size, sequence_length)*, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`. `1` for + tokens that are NOT MASKED, `0` for MASKED tokens. + use_rank_mode (*bool*, *optional*, defaults to *False*): + Whether to use rank mode or not. Rank mode is the decoding mode proposed in the paper. Please refer to + this line: https://github.com/salesforce/BLIP/blob/main/models/blip_vqa.py#L114 for more details. + **generate_kwargs: + Additional arguments passed to the *generate* function of the decoder Examples: @@ -1284,7 +1332,9 @@ def generate( @add_start_docstrings( """ - BLIP Vision Model with a projection layer on top (a linear layer on top of the pooled output). + BLIP Model with a vision and text projector, and a classification head on top. The model is used in the context of + image-text retrieval. Given an image and a text, the model returns the probability of the text being relevant to + the image. """, BLIP_START_DOCSTRING, ) @@ -1332,21 +1382,21 @@ def forward( ```python >>> from PIL import Image >>> import requests - >>> from transformers import CLIPProcessor, BLIPForImageCaptioning + >>> from transformers import BlipProcessor, BlipForImageTextRetrieval - >>> model = BLIPForImageCaptioning.from_pretrained("Salesforce/blip-image-captioning-base") - >>> processor = CLIPProcessor.from_pretrained("Salesforce/blip-image-captioning-base") + >>> model = BlipForImageTextRetrieval.from_pretrained("Salesforce/blip-image-captioning-base") + >>> processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) + >>> text = "an image of a cat" - >>> inputs = processor(images=image, return_tensors="pt") + >>> inputs = processor(images=image, text=text, return_tensors="pt") >>> outputs = model(**inputs) - >>> image_embeds = outputs.image_embeds + >>> print(outputs[0]) # similarity score ```""" return_dict = return_dict if return_dict is not None else self.config.use_return_dict - batch_size, _ = input_ids.shape vision_outputs = self.vision_model( pixel_values=pixel_values, @@ -1383,12 +1433,13 @@ def forward( output = image_feat @ text_feat.t() if not return_dict: - outputs = (output, vision_outputs[0]) + vision_outputs[2:] + outputs = (output, vision_outputs[0]) + vision_outputs[2:] + (question_embeds,) return tuple(output for output in outputs if output is not None) - return BlipTextVisionModelOutput( - image_embeds=output, + return BlipITMModelOutput( + itm_score=output, last_hidden_state=vision_outputs.last_hidden_state, hidden_states=vision_outputs.hidden_states, attentions=vision_outputs.attentions, + question_embeds=question_embeds, ) diff --git a/src/transformers/models/blip/processing_blip.py b/src/transformers/models/blip/processing_blip.py index c11b41efa0e1..9ea7a8e05b05 100644 --- a/src/transformers/models/blip/processing_blip.py +++ b/src/transformers/models/blip/processing_blip.py @@ -69,6 +69,34 @@ def __call__( Please refer to the docstring of the above two methods for more information. """ + # Get only text + if images is None: + if text is None: + raise ValueError("You have to specify either images or text.") + + self.current_processor = self.tokenizer + return self.tokenizer( + text=text, + add_special_tokens=add_special_tokens, + padding=padding, + truncation=truncation, + max_length=max_length, + stride=stride, + pad_to_multiple_of=pad_to_multiple_of, + return_token_type_ids=return_token_type_ids, + return_attention_mask=return_attention_mask, + return_overflowing_tokens=return_overflowing_tokens, + return_special_tokens_mask=return_special_tokens_mask, + return_offsets_mapping=return_offsets_mapping, + return_length=return_length, + verbose=verbose, + return_tensors=return_tensors, + **kwargs, + ) + + # add pixel_values + pixel_mask + encoding_feature_extractor = self.feature_extractor(images, return_tensors=return_tensors) + if text is not None: text_encoding = self.tokenizer( text=text, @@ -90,8 +118,7 @@ def __call__( ) else: text_encoding = None - # add pixel_values + pixel_mask - encoding_feature_extractor = self.feature_extractor(images, return_tensors=return_tensors) + if text_encoding is not None: encoding_feature_extractor.update(text_encoding) From ecf470b24dd49fabf6fb9d4f4f5d9d793d1589ba Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Mon, 12 Dec 2022 18:14:42 +0000 Subject: [PATCH 38/96] add integration tests --- src/transformers/models/blip/modeling_blip.py | 13 ++++++--- tests/models/blip/test_modeling_blip.py | 28 +++++++++++++++++-- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 796574e0bfec..883357cc5af9 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -1251,7 +1251,8 @@ def generate( input_ids: torch.LongTensor, pixel_values: torch.FloatTensor, attention_mask: Optional[torch.LongTensor] = None, - use_rank_mode: Optional[bool] = False, + pixel_mask: Optional[torch.LongTensor] = None, + token_type_ids: Optional[torch.LongTensor] = None, **generate_kwargs ) -> torch.LongTensor: r""" @@ -1265,9 +1266,11 @@ def generate( attention_mask (*torch.LongTensor* of shape *(batch_size, sequence_length)*, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`. `1` for tokens that are NOT MASKED, `0` for MASKED tokens. - use_rank_mode (*bool*, *optional*, defaults to *False*): - Whether to use rank mode or not. Rank mode is the decoding mode proposed in the paper. Please refer to - this line: https://github.com/salesforce/BLIP/blob/main/models/blip_vqa.py#L114 for more details. + pixel_mask (*torch.LongTensor* of shape *(batch_size, image_width, image_height)*, *optional*): + Mask to be used for the image - not used but kept for compatibility with the *BlipProcessor* + token_type_ids (*torch.LongTensor* of shape *(batch_size, sequence_length)*, *optional*): + Segment token indices to indicate first and second portions of the inputs. Not used but kept for + compatibility with the *BlipProcessor* **generate_kwargs: Additional arguments passed to the *generate* function of the decoder @@ -1370,6 +1373,8 @@ def forward( pixel_values: torch.FloatTensor, use_itm_head: Optional[bool] = True, attention_mask: Optional[torch.LongTensor] = None, + pixel_mask: Optional[torch.LongTensor] = None, + token_type_ids: Optional[torch.LongTensor] = None, output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, diff --git a/tests/models/blip/test_modeling_blip.py b/tests/models/blip/test_modeling_blip.py index 6f6de9fbd785..2ddf93fd1c2c 100644 --- a/tests/models/blip/test_modeling_blip.py +++ b/tests/models/blip/test_modeling_blip.py @@ -781,7 +781,7 @@ def test_inference_image_captioning(self): image = prepare_img() # image only - inputs = processor(image=image).to(torch_device) + inputs = processor(images=image).to(torch_device) predictions = model.generate(**inputs) @@ -801,7 +801,29 @@ def test_inference_image_captioning(self): ) def test_inference_vqa(self): - pass + model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base").to(torch_device) + processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base") + + image = prepare_img() + text = "how many dogs are in the picture?" + + inputs = processor(image, text=text, return_tensors="pt").to(torch_device) + out = model.generate(**inputs) + + # Test output + self.assertEqual(out[0].tolist(), [30522, 1015, 102]) def test_inference_itm(self): - pass + model = BlipForImageTextRetrieval.from_pretrained("Salesforce/blip-itm-base").to(torch_device) + processor = BlipProcessor.from_pretrained("Salesforce/blip-itm-base") + + image = prepare_img() + text = "A woman and her dog sitting in a beach" + + inputs = processor(image, text, return_tensors="pt").to(torch_device) + + out_itm = model(**inputs) + out = model(**inputs, use_itm_head=False) + + self.assertTrue(torch.allclose(out_itm[0].cpu(), torch.Tensor([[1.9452, -1.9378]]), atol=1e-3, rtol=1e-3)) + self.assertTrue(torch.allclose(out[0].cpu(), torch.Tensor([[0.5053]]), atol=1e-3, rtol=1e-3)) From 3dd7034c5f3adea9581a019aab937e8ebbc271ab Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Mon, 12 Dec 2022 18:34:23 +0000 Subject: [PATCH 39/96] add correct conversion script --- .../convert_blip_original_pytorch_to_hf.py | 228 +++++++++++------- 1 file changed, 137 insertions(+), 91 deletions(-) diff --git a/src/transformers/models/blip/convert_blip_original_pytorch_to_hf.py b/src/transformers/models/blip/convert_blip_original_pytorch_to_hf.py index e99ed2c3af9f..ddf9586b0e33 100644 --- a/src/transformers/models/blip/convert_blip_original_pytorch_to_hf.py +++ b/src/transformers/models/blip/convert_blip_original_pytorch_to_hf.py @@ -14,134 +14,180 @@ # limitations under the License. import argparse +import re import torch +from PIL import Image +from torchvision import transforms +from torchvision.transforms.functional import InterpolationMode + +import requests + +# git clone https://github.com/salesforce/BLIP.git +from models.blip import blip_decoder +from models.blip_itm import blip_itm +from models.blip_vqa import blip_vqa +from transformers import ( + BertTokenizer, + BlipConfig, + BlipForConditionalGeneration, + BlipForImageTextRetrieval, + BlipForQuestionAnswering, +) + + +def load_demo_image(image_size, device): + img_url = "https://storage.googleapis.com/sfr-vision-language-research/BLIP/demo.jpg" + raw_image = Image.open(requests.get(img_url, stream=True).raw).convert("RGB") + + w, h = raw_image.size + # display(raw_image.resize((w//5,h//5))) + + transform = transforms.Compose( + [ + transforms.Resize((image_size, image_size), interpolation=InterpolationMode.BICUBIC), + transforms.ToTensor(), + transforms.Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)), + ] + ) + image = transform(raw_image).unsqueeze(0).to(device) + return image + + +def rename_key(key): + if "visual_encoder" in key: + key = re.sub("visual_encoder*", "vision_model.encoder", key) + if "blocks" in key: + key = re.sub(r"blocks", "layers", key) + if "attn" in key: + key = re.sub(r"attn", "self_attn", key) + if "norm1" in key: + key = re.sub(r"norm1", "layer_norm1", key) + if "norm2" in key: + key = re.sub(r"norm2", "layer_norm2", key) + if "encoder.norm" in key: + key = re.sub(r"encoder.norm", "post_layernorm", key) + if "encoder.patch_embed.proj" in key: + key = re.sub(r"encoder.patch_embed.proj", "embeddings.patch_embedding", key) + + if "encoder.pos_embed" in key: + key = re.sub(r"encoder.pos_embed", "embeddings.position_embedding", key) + if "encoder.cls_token" in key: + key = re.sub(r"encoder.cls_token", "embeddings.class_embedding", key) + + if "self_attn" in key: + key = re.sub(r"self_attn.proj", "self_attn.projection", key) + + return key -from blip import load -from transformers import BLIPConfig, BLIPModel - - -def copy_attn_layer(hf_attn_layer, pt_attn_layer): - q_proj, k_proj, v_proj = pt_attn_layer.in_proj_weight.chunk(3, dim=0) - q_proj_bias, k_proj_bias, v_proj_bias = pt_attn_layer.in_proj_bias.chunk(3, dim=0) - - out_proj_weights = pt_attn_layer.out_proj.weight - out_proj_bias = pt_attn_layer.out_proj.bias - - hf_attn_layer.q_proj.weight.data = q_proj - hf_attn_layer.q_proj.bias.data = q_proj_bias - - hf_attn_layer.k_proj.weight.data = k_proj - hf_attn_layer.k_proj.bias.data = k_proj_bias - - hf_attn_layer.v_proj.weight.data = v_proj - hf_attn_layer.v_proj.bias.data = v_proj_bias - - hf_attn_layer.out_proj.weight = out_proj_weights - hf_attn_layer.out_proj.bias = out_proj_bias - - -def copy_mlp(hf_mlp, pt_mlp): - copy_linear(hf_mlp.fc1, pt_mlp.c_fc) - copy_linear(hf_mlp.fc2, pt_mlp.c_proj) +@torch.no_grad() +def convert_blip_checkpoint(pytorch_dump_folder_path, config_path=None): + """ + Copy/paste/tweak model's weights to transformers design. + """ + if config_path is not None: + config = BlipConfig.from_pretrained(config_path) + else: + config = BlipConfig(projection_dim=512, text_config={}, vision_config={}) -def copy_linear(hf_linear, pt_linear): - hf_linear.weight = pt_linear.weight - hf_linear.bias = pt_linear.bias + hf_model = BlipForConditionalGeneration(config).eval() + model_url = "https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_capfilt_large.pth" -def copy_layer(hf_layer, pt_layer): - # copy layer norms - copy_linear(hf_layer.layer_norm1, pt_layer.ln_1) - copy_linear(hf_layer.layer_norm2, pt_layer.ln_2) + pt_model = blip_decoder(pretrained=model_url, image_size=384, vit="base") + pt_model = pt_model.eval() - # copy MLP - copy_mlp(hf_layer.mlp, pt_layer.mlp) + modified_state_dict = pt_model.state_dict() + for key in modified_state_dict.copy(): + value = modified_state_dict.pop(key) + renamed_key = rename_key(key) + modified_state_dict[renamed_key] = value - # copy attn - copy_attn_layer(hf_layer.self_attn, pt_layer.attn) + hf_model.load_state_dict(modified_state_dict) + image_size = 384 + image = load_demo_image(image_size=image_size, device="cpu") + tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") + input_ids = tokenizer(["a picture of"]).input_ids -def copy_layers(hf_layers, pt_layers): - for hf_layer, pt_layer in zip(hf_layers, pt_layers): - copy_layer(hf_layer, pt_layer) + out = hf_model.generate(image, input_ids) + assert out[0].tolist() == [30522, 1037, 3861, 1997, 1037, 2450, 3564, 2006, 1996, 3509, 2007, 2014, 3899, 102] -def copy_encoder(hf_encoder, pt_model): - # copy embeds - hf_encoder.embeddings.token_embedding.weight = pt_model.token_embedding.weight - hf_encoder.embeddings.position_embedding.weight.data = pt_model.positional_embedding + out = hf_model.generate(image) - # copy layer norm - copy_linear(hf_encoder.final_layer_norm, pt_model.ln_final) + assert out[0].tolist() == [30522, 1037, 2450, 3564, 2006, 1996, 3509, 2007, 2014, 3899, 102] - # copy hidden layers - copy_layers(hf_encoder.encoder.layers, pt_model.transformer.resblocks) + if pytorch_dump_folder_path is not None: + hf_model.save_pretrained(pytorch_dump_folder_path) + # model_url = 'https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_vqa.pth' + model_url = ( + "https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_vqa_capfilt_large.pth" + ) -def copy_text_model_and_projection(hf_model, pt_model): - # copy projection - hf_model.text_projection.weight.data = pt_model.text_projection.data.T + vqa_model = blip_vqa(pretrained=model_url, image_size=image_size, vit="base") + vqa_model.eval() - # copy text encoder - copy_encoder(hf_model.text_model, pt_model) + modified_state_dict = vqa_model.state_dict() + for key in modified_state_dict.copy(): + value = modified_state_dict.pop(key) + renamed_key = rename_key(key) + modified_state_dict[renamed_key] = value + hf_vqa_model = BlipForQuestionAnswering(config) -def copy_vison_model_and_projection(hf_model, pt_model): - # copy projection - hf_model.visual_projection.weight.data = pt_model.visual.proj.data.T + hf_vqa_model.load_state_dict(modified_state_dict) - # copy layer norms - copy_linear(hf_model.vision_model.pre_layrnorm, pt_model.visual.ln_pre) - copy_linear(hf_model.vision_model.post_layernorm, pt_model.visual.ln_post) + question = ["How many dogs are in this image?"] + question_input_ids = tokenizer(question, return_tensors="pt").input_ids - # copy embeds - hf_model.vision_model.embeddings.patch_embedding.weight.data = pt_model.visual.conv1.weight.data - hf_model.vision_model.embeddings.class_embedding = pt_model.visual.class_embedding - hf_model.vision_model.embeddings.position_embedding.weight.data = pt_model.visual.positional_embedding.data + answer = hf_vqa_model.generate(question_input_ids, image) + print(tokenizer.decode(answer[0])) - # copy encoder - copy_layers(hf_model.vision_model.encoder.layers, pt_model.visual.transformer.resblocks) + assert tokenizer.decode(answer[0]) == "[UNK] 1 [SEP]" + if pytorch_dump_folder_path is not None: + hf_vqa_model.save_pretrained(pytorch_dump_folder_path + "_vqa") + model_url = "https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_retrieval_coco.pth" -@torch.no_grad() -def convert_blip_checkpoint(checkpoint_path, pytorch_dump_folder_path, config_path=None): - """ - Copy/paste/tweak model's weights to transformers design. - """ - if config_path is not None: - config = BLIPConfig.from_pretrained(config_path) - else: - config = BLIPConfig(projection_dim=512, text_config={}, vision_config={}) + itm_model = blip_itm(pretrained=model_url, image_size=image_size, vit="base") + itm_model.eval() - hf_model = BLIPModel(config).eval() + modified_state_dict = itm_model.state_dict() + for key in modified_state_dict.copy(): + value = modified_state_dict.pop(key) + renamed_key = rename_key(key) + modified_state_dict[renamed_key] = value - pt_model, _ = load(checkpoint_path, device="cpu", jit=False) - pt_model = pt_model.eval() + hf_itm_model = BlipForImageTextRetrieval(config) - copy_text_model_and_projection(hf_model, pt_model) - copy_vison_model_and_projection(hf_model, pt_model) - hf_model.logit_scale = pt_model.logit_scale + question = ["A picture of a woman with a dog sitting in a beach"] + question_input_ids = tokenizer( + question, + return_tensors="pt", + padding="max_length", + truncation=True, + max_length=35, + ).input_ids - input_ids = torch.arange(0, 77).unsqueeze(0) - pixel_values = torch.randn(1, 3, 224, 224) + hf_itm_model.load_state_dict(modified_state_dict) + hf_itm_model.eval() - hf_logits_per_image, hf_logits_per_text = hf_model( - input_ids=input_ids, pixel_values=pixel_values, return_dict=True - )[1:3] - pt_logits_per_image, pt_logits_per_text = pt_model(pixel_values, input_ids) + out_itm = hf_itm_model(question_input_ids, image, use_itm_head=True) + out = hf_itm_model(question_input_ids, image, use_itm_head=False) - assert torch.allclose(hf_logits_per_image, pt_logits_per_image, atol=1e-3) - assert torch.allclose(hf_logits_per_text, pt_logits_per_text, atol=1e-3) + assert out[0].item() == 0.2110687494277954 + assert torch.nn.functional.softmax(out_itm[0], dim=1)[:, 1].item() == 0.45698845386505127 - hf_model.save_pretrained(pytorch_dump_folder_path) + if pytorch_dump_folder_path is not None: + hf_itm_model.save_pretrained(pytorch_dump_folder_path + "_itm") if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model.") - parser.add_argument("--checkpoint_path", default=None, type=str, help="Path to fairseq checkpoint") parser.add_argument("--config_path", default=None, type=str, help="Path to hf config.json of model to convert") args = parser.parse_args() From 671c79681a5190a43435656aa6140631253d137d Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 13 Dec 2022 00:03:55 +0100 Subject: [PATCH 40/96] Apply suggestions from code review Co-authored-by: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> --- src/transformers/models/auto/modeling_auto.py | 2 +- src/transformers/models/blip/image_processing_blip.py | 2 +- src/transformers/models/blip/modeling_blip.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/auto/modeling_auto.py b/src/transformers/models/auto/modeling_auto.py index 465e42b1fc83..91b398804d81 100644 --- a/src/transformers/models/auto/modeling_auto.py +++ b/src/transformers/models/auto/modeling_auto.py @@ -850,7 +850,7 @@ _MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES = OrderedDict( [ # Model for Zero Shot Image Classification mapping - ("blip", "BLIPModel"), + ("blip", "BlipModel"), ("chinese_clip", "ChineseCLIPModel"), ("clip", "CLIPModel"), ("clipseg", "CLIPSegModel"), diff --git a/src/transformers/models/blip/image_processing_blip.py b/src/transformers/models/blip/image_processing_blip.py index e19b84cb290c..0fcbe151543b 100644 --- a/src/transformers/models/blip/image_processing_blip.py +++ b/src/transformers/models/blip/image_processing_blip.py @@ -164,7 +164,7 @@ class BlipImageProcessor(BaseImageProcessor): do_resize (`bool`, *optional*, defaults to `True`): Whether to resize the image's (height, width) dimensions to the specified `size`. Can be overridden by the `do_resize` parameter in the `preprocess` method. - size (`Dict[str, int]` *optional*, defaults to `{"shortest_edge": 384}`): + size (`Dict[str, int]`, *optional*, defaults to `{"shortest_edge": 384}`): Resize the shorter side of the input to `size["shortest_edge"]`. The longer side will be limited to under `int((1333 / 800) * size["shortest_edge"])` while preserving the aspect ratio. Only has an effect if `do_resize` is set to `True`. Can be overridden by the `size` parameter in the `preprocess` method. diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 883357cc5af9..677771782407 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -1281,7 +1281,7 @@ def generate( >>> import requests >>> from transformers import BlipTokenizer, BlipForImageCaptioning - >>> model = BLIPForImageCaptioning.from_pretrained("Salesforce/blip-image-captioning-base") + >>> model = BlipForImageCaptioning.from_pretrained("Salesforce/blip-image-captioning-base") >>> processor = CLIPProcessor.from_pretrained("Salesforce/blip-image-captioning-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" From 9afcd0efd9de462c1b5f13574bd031974dbcc4ec Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Mon, 12 Dec 2022 23:10:29 +0000 Subject: [PATCH 41/96] add `blip` to tokenization auto --- src/transformers/models/auto/tokenization_auto.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/models/auto/tokenization_auto.py b/src/transformers/models/auto/tokenization_auto.py index e2fa1ddd7ce6..1385116a43b4 100644 --- a/src/transformers/models/auto/tokenization_auto.py +++ b/src/transformers/models/auto/tokenization_auto.py @@ -77,6 +77,7 @@ ("biogpt", ("BioGptTokenizer", None)), ("blenderbot", ("BlenderbotTokenizer", "BlenderbotTokenizerFast")), ("blenderbot-small", ("BlenderbotSmallTokenizer", None)), + ("blip", ("BertTokenizer", "BertTokenizerFast" if is_tokenizers_available() else None)), ("bloom", (None, "BloomTokenizerFast" if is_tokenizers_available() else None)), ("byt5", ("ByT5Tokenizer", None)), ( From 829f2b356204f6f7f83bb744c3711137b2d51232 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Mon, 12 Dec 2022 23:21:33 +0000 Subject: [PATCH 42/96] fix docstrings --- .../models/blip/image_processing_blip.py | 8 ++---- src/transformers/models/blip/modeling_blip.py | 25 +++++++++++-------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/transformers/models/blip/image_processing_blip.py b/src/transformers/models/blip/image_processing_blip.py index 0fcbe151543b..a937426d5a91 100644 --- a/src/transformers/models/blip/image_processing_blip.py +++ b/src/transformers/models/blip/image_processing_blip.py @@ -164,10 +164,8 @@ class BlipImageProcessor(BaseImageProcessor): do_resize (`bool`, *optional*, defaults to `True`): Whether to resize the image's (height, width) dimensions to the specified `size`. Can be overridden by the `do_resize` parameter in the `preprocess` method. - size (`Dict[str, int]`, *optional*, defaults to `{"shortest_edge": 384}`): - Resize the shorter side of the input to `size["shortest_edge"]`. The longer side will be limited to under - `int((1333 / 800) * size["shortest_edge"])` while preserving the aspect ratio. Only has an effect if - `do_resize` is set to `True`. Can be overridden by the `size` parameter in the `preprocess` method. + size (`Dict[str, int]`, *optional*, defaults to `384`): + Resize the input to the given size. Only has an effect if `do_resize` is set to `True`. size_divisor (`int`, *optional*, defaults to 32): The size by which to make sure both the height and width can be divided. Only has an effect if `do_resize` is set to `True`. Can be overridden by the `size_divisor` parameter in the `preprocess` method. @@ -250,8 +248,6 @@ def resize( Image to resize. size (`Dict[str, int]`): Controls the size of the output image. Should be of the form `{"shortest_edge": int}`. - size_divisor (`int`, defaults to 32): - The image is resized to a size that is a multiple of this value. resample (`PILImageResampling` filter, *optional*, defaults to `PILImageResampling.BICUBIC`): Resampling filter to use when resiizing the image. data_format (`str` or `ChannelDimension`, *optional*): diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 677771782407..64a041db052a 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -71,20 +71,22 @@ class BlipForConditionalGenerationModelOutput(ModelOutput): last hidden states. This class also adds the loss term from the text decoder. Args: - loss (*optional*, *optional* returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`): + loss (`torch.FloatTensor`, *optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`): Languge modeling loss from the text decoder. - image_embeds (*torch.FloatTensor* of shape *(batch_size, output_dim)* *optional* returned when model is initialized with *with_projection=True*): - The image embeddings obtained by applying the projection layer to the pooler_output. - last_hidden_state (*torch.FloatTensor* of shape *(batch_size, sequence_length, hidden_size)*): + image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)`, *optional*): + The image embeddings obtained after applying the Vision Transformer model to the input image. + decoder_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`, *optional*): + Prediction scores of the language modeling head of the text decoder model. + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the model. - hidden_states (*tuple(torch.FloatTensor)*, *optional*, returned when *output_hidden_states=True* is passed or when *config.output_hidden_states=True*): - Tuple of *torch.FloatTensor* (one for the output of the embeddings, if the model has an embedding layer, + - one for the output of each layer) of shape *(batch_size, sequence_length, hidden_size)*. + hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. - attentions (*tuple(torch.FloatTensor)*, *optional*, returned when *output_attentions=True* is passed or when *config.output_attentions=True*): - Tuple of *torch.FloatTensor* (one for each layer) of shape *(batch_size, num_heads, sequence_length, - sequence_length)*. + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. @@ -577,7 +579,8 @@ class BlipEncoder(nn.Module): [`BlipEncoderLayer`]. Args: - config: BlipConfig + config (`BlipConfig`): + The corresponding vision configuration for the `BlipEncoder`. """ def __init__(self, config: BlipConfig): From 27437be69ff44926e4b3b456aea9cb40ebd83a06 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 19:34:14 +0000 Subject: [PATCH 43/96] fix test + add image --- docs/source/en/model_doc/blip.mdx | 2 ++ tests/models/blip/test_modeling_blip.py | 9 +++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/source/en/model_doc/blip.mdx b/docs/source/en/model_doc/blip.mdx index 447835860ece..de42029b4476 100644 --- a/docs/source/en/model_doc/blip.mdx +++ b/docs/source/en/model_doc/blip.mdx @@ -26,6 +26,8 @@ The abstract from the paper is the following: *Vision-Language Pre-training (VLP) has advanced the performance for many vision-language tasks. However, most existing pre-trained models only excel in either understanding-based tasks or generation-based tasks. Furthermore, performance improvement has been largely achieved by scaling up the dataset with noisy image-text pairs collected from the web, which is a suboptimal source of supervision. In this paper, we propose BLIP, a new VLP framework which transfers flexibly to both vision-language understanding and generation tasks. BLIP effectively utilizes the noisy web data by bootstrapping the captions, where a captioner generates synthetic captions and a filter removes the noisy ones. We achieve state-of-the-art results on a wide range of vision-language tasks, such as image-text retrieval (+2.7% in average recall@1), image captioning (+2.8% in CIDEr), and VQA (+1.6% in VQA score). BLIP also demonstrates strong generalization ability when directly transferred to videolanguage tasks in a zero-shot manner. Code, models, and datasets are released.* +![BLIP.gif](https://s3.amazonaws.com/moonup/production/uploads/1670928184033-62441d1d9fdefb55a0b7d12c.gif) + This model was contributed by [ybelkada](https://huggingface.co/ybelkada). The original code can be found [here](https://github.com/salesforce/BLIP). diff --git a/tests/models/blip/test_modeling_blip.py b/tests/models/blip/test_modeling_blip.py index 2ddf93fd1c2c..dedcfce5a314 100644 --- a/tests/models/blip/test_modeling_blip.py +++ b/tests/models/blip/test_modeling_blip.py @@ -585,6 +585,7 @@ class BlipTextImageModelTest(ModelTesterMixin, unittest.TestCase): test_pruning = False test_resize_embeddings = False test_attention_outputs = False + test_torchscript = False def setUp(self): self.model_tester = BlipTextImageModelsModelTester(self) @@ -776,12 +777,12 @@ def prepare_img(): @slow class BlipModelIntegrationTest(unittest.TestCase): def test_inference_image_captioning(self): - model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base").to(torch_device) - processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") + model = BlipForConditionalGeneration.from_pretrained("ybelkada/blip-image-captioning-base").to(torch_device) + processor = BlipProcessor.from_pretrained("ybelkada/blip-image-captioning-base") image = prepare_img() # image only - inputs = processor(images=image).to(torch_device) + inputs = processor(images=image, return_tensors="pt").to(torch_device) predictions = model.generate(**inputs) @@ -790,7 +791,7 @@ def test_inference_image_captioning(self): # image and context context = ["a picture of"] - inputs = processor(image=image, text=context).to(torch_device) + inputs = processor(images=image, text=context, return_tensors="pt").to(torch_device) predictions = model.generate(**inputs) From f5978b732fae96ea694651482eb7a3c0f8c8267f Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 19:36:53 +0000 Subject: [PATCH 44/96] remove processor from uncorrect place --- src/transformers/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py index 0891a229181d..d0db90405028 100644 --- a/src/transformers/__init__.py +++ b/src/transformers/__init__.py @@ -755,7 +755,7 @@ _import_structure["image_utils"] = ["ImageFeatureExtractionMixin"] _import_structure["models.beit"].extend(["BeitFeatureExtractor", "BeitImageProcessor"]) _import_structure["models.bit"].extend(["BitImageProcessor"]) - _import_structure["models.blip"].extend(["BlipImageProcessor", "BlipProcessor"]) + _import_structure["models.blip"].extend(["BlipImageProcessor"]) _import_structure["models.chinese_clip"].extend(["ChineseCLIPFeatureExtractor", "ChineseCLIPImageProcessor"]) _import_structure["models.clip"].extend(["CLIPFeatureExtractor", "CLIPImageProcessor"]) _import_structure["models.conditional_detr"].extend( @@ -3971,7 +3971,7 @@ from .image_utils import ImageFeatureExtractionMixin from .models.beit import BeitFeatureExtractor, BeitImageProcessor from .models.bit import BitImageProcessor - from .models.blip import BlipImageProcessor, BlipProcessor + from .models.blip import BlipImageProcessor from .models.chinese_clip import ChineseCLIPFeatureExtractor, ChineseCLIPImageProcessor from .models.clip import CLIPFeatureExtractor, CLIPImageProcessor from .models.conditional_detr import ConditionalDetrFeatureExtractor, ConditionalDetrImageProcessor From 256d02a74a9fcb8c156e852e46718a15e4e133ac Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 13 Dec 2022 20:39:20 +0100 Subject: [PATCH 45/96] Apply suggestions from code review Co-authored-by: NielsRogge <48327001+NielsRogge@users.noreply.github.com> --- src/transformers/models/blip/configuration_blip.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/transformers/models/blip/configuration_blip.py b/src/transformers/models/blip/configuration_blip.py index 4ff09b0dbbf5..11721ebaa326 100644 --- a/src/transformers/models/blip/configuration_blip.py +++ b/src/transformers/models/blip/configuration_blip.py @@ -52,8 +52,8 @@ class BlipTextConfig(PretrainedConfig): r""" - This is the configuration class to store the configuration of a [`BlipTextModel`]. It is used to instantiate an - `BlipText` model according to the specified arguments, defining the model architecture. Instantiating a + This is the configuration class to store the configuration of a [`BlipTextModel`]. It is used to instantiate a + BLIP text model according to the specified arguments, defining the model architecture. Instantiating a configuration with the defaults will yield a similar configuration to that of the `BlipText` used by the [base architectures](https://huggingface.co/Salesforce/blip-vqa-base). @@ -186,7 +186,7 @@ def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], class BlipVisionConfig(PretrainedConfig): r""" This is the configuration class to store the configuration of a [`BlipVisionModel`]. It is used to instantiate a - Blip vision model according to the specified arguments, defining the model architecture. Instantiating a + BLIP vision model according to the specified arguments, defining the model architecture. Instantiating a configuration defaults will yield a similar configuration to that of the Blip-base [Salesforce/blip-vqa-base](https://huggingface.co/Salesforce/blip-vqa-base) architecture. @@ -294,8 +294,8 @@ def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], class BlipConfig(PretrainedConfig): r""" [`BlipConfig`] is the configuration class to store the configuration of a [`BlipModel`]. It is used to instantiate - Blip model according to the specified arguments, defining the text model and vision model configs. Instantiating a - configuration with the defaults will yield a similar configuration to that of the Blip-base + a BLIP model according to the specified arguments, defining the text model and vision model configs. Instantiating a + configuration with the defaults will yield a similar configuration to that of the BLIP-base [Salesforce/blip-vqa-base](https://huggingface.co/Salesforce/blip-vqa-base) architecture. Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the From 0696a41b6858ea4370c4fe9c91345aa09a580e5b Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 19:41:05 +0000 Subject: [PATCH 46/96] clean up a bit --- .../models/blip/convert_blip_original_pytorch_to_hf.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/transformers/models/blip/convert_blip_original_pytorch_to_hf.py b/src/transformers/models/blip/convert_blip_original_pytorch_to_hf.py index ddf9586b0e33..9deda9c11609 100644 --- a/src/transformers/models/blip/convert_blip_original_pytorch_to_hf.py +++ b/src/transformers/models/blip/convert_blip_original_pytorch_to_hf.py @@ -40,9 +40,6 @@ def load_demo_image(image_size, device): img_url = "https://storage.googleapis.com/sfr-vision-language-research/BLIP/demo.jpg" raw_image = Image.open(requests.get(img_url, stream=True).raw).convert("RGB") - w, h = raw_image.size - # display(raw_image.resize((w//5,h//5))) - transform = transforms.Compose( [ transforms.Resize((image_size, image_size), interpolation=InterpolationMode.BICUBIC), From c5b92519e66aafec38f332d9947e275c7808bbee Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 13 Dec 2022 20:42:26 +0100 Subject: [PATCH 47/96] Apply suggestions from code review Co-authored-by: NielsRogge <48327001+NielsRogge@users.noreply.github.com> --- src/transformers/models/blip/image_processing_blip.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/blip/image_processing_blip.py b/src/transformers/models/blip/image_processing_blip.py index a937426d5a91..be92858f5af7 100644 --- a/src/transformers/models/blip/image_processing_blip.py +++ b/src/transformers/models/blip/image_processing_blip.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Feature extractor class for ViLT.""" +"""Image processor class for BLIP.""" import warnings from typing import Any, Dict, Iterable, List, Optional, Tuple, Union @@ -158,7 +158,7 @@ def get_resize_output_image_size( class BlipImageProcessor(BaseImageProcessor): r""" - Constructs a ViLT image processor. + Constructs a BLIP image processor. Args: do_resize (`bool`, *optional*, defaults to `True`): From f6636b92931772c52e8964eaab3707d23394d069 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 19:45:36 +0000 Subject: [PATCH 48/96] clean pixel mask --- .../models/blip/image_processing_blip.py | 28 +------------------ 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/src/transformers/models/blip/image_processing_blip.py b/src/transformers/models/blip/image_processing_blip.py index a937426d5a91..226706c6e041 100644 --- a/src/transformers/models/blip/image_processing_blip.py +++ b/src/transformers/models/blip/image_processing_blip.py @@ -95,23 +95,6 @@ def pad( return padded_image -# Copied from transformers.models.vilt.image_processing_vilt.make_pixel_mask -def make_pixel_mask(image: np.ndarray, output_size: Tuple[int, int]) -> np.ndarray: - """ - Make a pixel mask for the image, where 1 indicates a valid pixel and 0 indicates padding. - - Args: - image (`np.ndarray`): - Image to make the pixel mask for. - output_size (`Tuple[int, int]`): - Output size of the mask. - """ - input_height, input_width = get_image_size(image) - mask = np.zeros(output_size, dtype=np.int64) - mask[:input_height, :input_width] = 1 - return mask - - # Copied from transformers.models.vilt.image_processing_vilt.get_max_dimensions def get_max_dimensions(images: List[np.ndarray]) -> List[int]: """ @@ -210,8 +193,6 @@ def __init__( do_pad: bool = True, **kwargs ) -> None: - if "pad_and_return_pixel_mask" in kwargs: - do_pad = kwargs.pop("pad_and_return_pixel_mask") super().__init__(**kwargs) size = size if size is not None else 384 @@ -303,7 +284,6 @@ def normalize( def pad( self, images: List[np.ndarray], - return_pixel_mask: bool = True, return_tensors: Optional[Union[str, TensorType]] = None, data_format: Optional[ChannelDimension] = None, ) -> BatchFeature: @@ -314,8 +294,6 @@ def pad( Args: images (`List[np.ndarray]`): Batch of images to pad. - return_pixel_mask (`bool`, *optional*, defaults to `False`): - Whether to return the pixel mask. return_tensors (`str` or `TensorType`, *optional*): The type of tensors to return. Can be one of: - Unset: Return a list of `np.ndarray`. @@ -329,9 +307,6 @@ def pad( pad_size = get_max_dimensions(images) padded_images = [pad(image=image, output_size=pad_size, data_format=data_format) for image in images] data = {"pixel_values": padded_images} - if return_pixel_mask: - masks = [make_pixel_mask(image=image, output_size=pad_size) for image in images] - data["pixel_mask"] = masks return BatchFeature(data=data, tensor_type=return_tensors) @@ -365,7 +340,6 @@ def pad_and_create_pixel_mask( images = [to_numpy_array(image) for image in pixel_values_list] return self.pad( images=images, - return_pixel_mask=True, return_tensors=return_tensors, data_format=data_format, ) @@ -475,7 +449,7 @@ def preprocess( images = [to_channel_dimension_format(image, data_format) for image in images] if do_pad: - encoded_outputs = self.pad(images, return_pixel_mask=True, return_tensors=return_tensors) + encoded_outputs = self.pad(images, return_tensors=return_tensors) else: encoded_outputs = BatchFeature(data={"pixel_values": images}, tensor_type=return_tensors) From a69d8bc00ff6cd80942896700c4d9fa643fe8aad Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 19:45:50 +0000 Subject: [PATCH 49/96] clean pixel mask --- src/transformers/models/blip/modeling_blip.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 64a041db052a..fd928d43eb43 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -1061,7 +1061,6 @@ def generate( self, pixel_values: torch.FloatTensor, input_ids: Optional[torch.LongTensor] = None, - pixel_mask: Optional[torch.LongTensor] = None, token_type_ids: Optional[torch.LongTensor] = None, attention_mask: Optional[torch.LongTensor] = None, **generate_kwargs @@ -1074,8 +1073,6 @@ def generate( Input image to be processed input_ids (*torch.LongTensor* of shape *(batch_size, sequence_length)*, *optional*): The sequence used as a prompt for the generation. - pixel_mask (*torch.LongTensor* of shape *(batch_size, image_width, image_height)*, *optional*): - Mask to be used for the image - not used but kept for compatibility with the *BlipProcessor* token_type_ids (*torch.LongTensor* of shape *(batch_size, sequence_length)*, *optional*): Segment token indices to indicate first and second portions of the inputs. Not used but kept for compatibility with the *BlipProcessor* @@ -1254,7 +1251,6 @@ def generate( input_ids: torch.LongTensor, pixel_values: torch.FloatTensor, attention_mask: Optional[torch.LongTensor] = None, - pixel_mask: Optional[torch.LongTensor] = None, token_type_ids: Optional[torch.LongTensor] = None, **generate_kwargs ) -> torch.LongTensor: @@ -1269,8 +1265,6 @@ def generate( attention_mask (*torch.LongTensor* of shape *(batch_size, sequence_length)*, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`. `1` for tokens that are NOT MASKED, `0` for MASKED tokens. - pixel_mask (*torch.LongTensor* of shape *(batch_size, image_width, image_height)*, *optional*): - Mask to be used for the image - not used but kept for compatibility with the *BlipProcessor* token_type_ids (*torch.LongTensor* of shape *(batch_size, sequence_length)*, *optional*): Segment token indices to indicate first and second portions of the inputs. Not used but kept for compatibility with the *BlipProcessor* @@ -1376,7 +1370,6 @@ def forward( pixel_values: torch.FloatTensor, use_itm_head: Optional[bool] = True, attention_mask: Optional[torch.LongTensor] = None, - pixel_mask: Optional[torch.LongTensor] = None, token_type_ids: Optional[torch.LongTensor] = None, output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, From 7fdf5e4b26b6d489d9f64649d2dc5ef541ab7e7f Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 19:48:08 +0000 Subject: [PATCH 50/96] fix `F` --- src/transformers/models/blip/modeling_blip.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index fd928d43eb43..4e37396ade2c 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -17,7 +17,7 @@ from typing import Any, Optional, Tuple, Union import torch -import torch.nn.functional as F +from torch.nn.functional import normalize import torch.utils.checkpoint from torch import nn @@ -1428,8 +1428,8 @@ def forward( ) question_embeds = question_embeds[0] if not return_dict else question_embeds.last_hidden_state - image_feat = F.normalize(self.vision_proj(image_embeds[:, 0, :]), dim=-1) - text_feat = F.normalize(self.text_proj(question_embeds[:, 0, :]), dim=-1) + image_feat = normalize(self.vision_proj(image_embeds[:, 0, :]), dim=-1) + text_feat = normalize(self.text_proj(question_embeds[:, 0, :]), dim=-1) output = image_feat @ text_feat.t() From e644cb5f1a3a09c1ad661a07418d9b16eb85da7a Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 13 Dec 2022 20:48:53 +0100 Subject: [PATCH 51/96] Update src/transformers/models/blip/modeling_blip.py Co-authored-by: NielsRogge <48327001+NielsRogge@users.noreply.github.com> --- src/transformers/models/blip/modeling_blip.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 4e37396ade2c..b386f0fb3e46 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """ PyTorch BLIP model.""" + from dataclasses import dataclass from typing import Any, Optional, Tuple, Union From 537d552c4849c6d0a01116bea24fd9bceb6c084d Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 13 Dec 2022 20:49:28 +0100 Subject: [PATCH 52/96] Apply suggestions from code review Co-authored-by: NielsRogge <48327001+NielsRogge@users.noreply.github.com> --- src/transformers/models/blip/modeling_blip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index b386f0fb3e46..dd0f9bb7a384 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -135,7 +135,7 @@ class BlipTextVisionModelOutput(ModelOutput): @dataclass -class BlipITMModelOutput(ModelOutput): +class BlipImageTextMatchingModelOutput(ModelOutput): """ Adapted from the base class for vision model's outputs that also contains image embeddings of the pooling of the last hidden states. This class also adds the loss term from the text decoder as well as the image-text similarity From b9b3d5188ece8272a26681e59fabfdc7d86f048a Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 19:50:02 +0000 Subject: [PATCH 53/96] fix output --- src/transformers/models/blip/modeling_blip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index dd0f9bb7a384..cf760daf9632 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -1438,7 +1438,7 @@ def forward( outputs = (output, vision_outputs[0]) + vision_outputs[2:] + (question_embeds,) return tuple(output for output in outputs if output is not None) - return BlipITMModelOutput( + return BlipImageTextMatchingModelOutput( itm_score=output, last_hidden_state=vision_outputs.last_hidden_state, hidden_states=vision_outputs.hidden_states, From e8b6118b26aecd19c9d06548506a6d21ec0c867c Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 13 Dec 2022 20:57:23 +0100 Subject: [PATCH 54/96] Apply suggestions from code review Co-authored-by: NielsRogge <48327001+NielsRogge@users.noreply.github.com> --- src/transformers/models/blip/modeling_blip.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index cf760daf9632..57f77e1740b5 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -961,7 +961,7 @@ def forward( @add_start_docstrings( """ - BLIP Model trained to generate captions on images. The model can also take as input a text input. The text decoder + BLIP Model for image captioning. The model consists of a vision encoder and a text decoder. One can optionally pass `input_ids` to the model, which serve as a text prompt, to make the text decoder continue the prompt. Otherwise, the decoder starts generating text from the [BOS] (beginning-of-sequence) token. will start generating the caption from the text input. If no text input is provided, the decoder will start with the [BOS] token only. """, @@ -1008,8 +1008,8 @@ def forward( >>> import requests >>> from transformers import BlipProcessor, BlipForConditionalGeneration - >>> model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base") >>> processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") + >>> model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) From 895b0721bf02fef2bbb7d83b3011568e4772ef10 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 20:01:24 +0000 Subject: [PATCH 55/96] fix pad token id --- src/transformers/models/blip/modeling_blip.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 57f77e1740b5..b066811c3009 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -980,6 +980,7 @@ def __init__(self, config: BlipConfig): self.text_decoder = BlipTextLMHeadModel(config.text_config) self.decoder_input_ids = config.text_config.bos_token_id + self.decoder_pad_token_id = config.text_config.pad_token_id # Initialize weights and apply final processing self.post_init() @@ -997,7 +998,7 @@ def forward( output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, - ) -> Union[Tuple, BlipTextVisionModelOutput]: + ) -> Union[Tuple, BlipForConditionalGenerationModelOutput]: r""" Returns: @@ -1034,7 +1035,7 @@ def forward( if input_ids is None: input_ids = torch.LongTensor([[self.decoder_input_ids] * batch_size]).to(image_embeds.device) - decoder_targets = input_ids.masked_fill(input_ids == self.decoder_input_ids, -100) + decoder_targets = input_ids.masked_fill(input_ids == self.decoder_pad_token_id, -100) outputs = self.text_decoder( input_ids=input_ids, From e018a35b4c7f564ce7d52640bc999852905b0026 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 20:06:43 +0000 Subject: [PATCH 56/96] remove `token_type_ids` --- src/transformers/models/blip/modeling_blip.py | 22 ++++++++----------- .../models/blip/processing_blip.py | 8 +++---- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index b066811c3009..4c6a70e84845 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -18,9 +18,9 @@ from typing import Any, Optional, Tuple, Union import torch -from torch.nn.functional import normalize import torch.utils.checkpoint from torch import nn +from torch.nn.functional import normalize from ...activations import ACT2FN from ...modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling @@ -961,9 +961,10 @@ def forward( @add_start_docstrings( """ - BLIP Model for image captioning. The model consists of a vision encoder and a text decoder. One can optionally pass `input_ids` to the model, which serve as a text prompt, to make the text decoder continue the prompt. Otherwise, the decoder starts generating text from the [BOS] (beginning-of-sequence) token. - will start generating the caption from the text input. If no text input is provided, the decoder will start with - the [BOS] token only. + BLIP Model for image captioning. The model consists of a vision encoder and a text decoder. One can optionally pass + `input_ids` to the model, which serve as a text prompt, to make the text decoder continue the prompt. Otherwise, + the decoder starts generating text from the [BOS] (beginning-of-sequence) token. will start generating the caption + from the text input. If no text input is provided, the decoder will start with the [BOS] token only. """, BLIP_START_DOCSTRING, ) @@ -1010,7 +1011,7 @@ def forward( >>> from transformers import BlipProcessor, BlipForConditionalGeneration >>> processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") - >>> model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base") + >>> model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) @@ -1063,7 +1064,6 @@ def generate( self, pixel_values: torch.FloatTensor, input_ids: Optional[torch.LongTensor] = None, - token_type_ids: Optional[torch.LongTensor] = None, attention_mask: Optional[torch.LongTensor] = None, **generate_kwargs ) -> torch.LongTensor: @@ -1075,9 +1075,6 @@ def generate( Input image to be processed input_ids (*torch.LongTensor* of shape *(batch_size, sequence_length)*, *optional*): The sequence used as a prompt for the generation. - token_type_ids (*torch.LongTensor* of shape *(batch_size, sequence_length)*, *optional*): - Segment token indices to indicate first and second portions of the inputs. Not used but kept for - compatibility with the *BlipProcessor* attention_mask (*torch.LongTensor* of shape *(batch_size, sequence_length)*, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: @@ -1086,9 +1083,9 @@ def generate( ```python >>> from PIL import Image >>> import requests - >>> from transformers import BlipProcessor, BlipForImageCaptioning + >>> from transformers import BlipProcessor, BlipForConditionalGeneration - >>> model = BlipForImageCaptioning.from_pretrained("Salesforce/blip-image-captioning-base") + >>> model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base") >>> processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" @@ -1096,8 +1093,7 @@ def generate( >>> inputs = processor(images=image, return_tensors="pt") - >>> outputs = model(**inputs) - >>> image_embeds = outputs.image_embeds + >>> outputs = model.generate(**inputs) ``` """ diff --git a/src/transformers/models/blip/processing_blip.py b/src/transformers/models/blip/processing_blip.py index 9ea7a8e05b05..b609961de63e 100644 --- a/src/transformers/models/blip/processing_blip.py +++ b/src/transformers/models/blip/processing_blip.py @@ -53,7 +53,6 @@ def __call__( max_length: Optional[int] = None, stride: int = 0, pad_to_multiple_of: Optional[int] = None, - return_token_type_ids: Optional[bool] = None, return_attention_mask: Optional[bool] = None, return_overflowing_tokens: bool = False, return_special_tokens_mask: bool = False, @@ -75,7 +74,7 @@ def __call__( raise ValueError("You have to specify either images or text.") self.current_processor = self.tokenizer - return self.tokenizer( + text_encoding = self.tokenizer( text=text, add_special_tokens=add_special_tokens, padding=padding, @@ -83,7 +82,6 @@ def __call__( max_length=max_length, stride=stride, pad_to_multiple_of=pad_to_multiple_of, - return_token_type_ids=return_token_type_ids, return_attention_mask=return_attention_mask, return_overflowing_tokens=return_overflowing_tokens, return_special_tokens_mask=return_special_tokens_mask, @@ -93,6 +91,8 @@ def __call__( return_tensors=return_tensors, **kwargs, ) + _ = text_encoding.pop("token_type_ids", None) + return text_encoding # add pixel_values + pixel_mask encoding_feature_extractor = self.feature_extractor(images, return_tensors=return_tensors) @@ -106,7 +106,6 @@ def __call__( max_length=max_length, stride=stride, pad_to_multiple_of=pad_to_multiple_of, - return_token_type_ids=return_token_type_ids, return_attention_mask=return_attention_mask, return_overflowing_tokens=return_overflowing_tokens, return_special_tokens_mask=return_special_tokens_mask, @@ -116,6 +115,7 @@ def __call__( return_tensors=return_tensors, **kwargs, ) + _ = text_encoding.pop("token_type_ids", None) else: text_encoding = None From 2312724317e9db0bcc9ce14a4bb8328f313cc066 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 20:06:48 +0000 Subject: [PATCH 57/96] make fixup --- src/transformers/models/blip/configuration_blip.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/transformers/models/blip/configuration_blip.py b/src/transformers/models/blip/configuration_blip.py index 11721ebaa326..b450f839e4f4 100644 --- a/src/transformers/models/blip/configuration_blip.py +++ b/src/transformers/models/blip/configuration_blip.py @@ -52,9 +52,9 @@ class BlipTextConfig(PretrainedConfig): r""" - This is the configuration class to store the configuration of a [`BlipTextModel`]. It is used to instantiate a - BLIP text model according to the specified arguments, defining the model architecture. Instantiating a - configuration with the defaults will yield a similar configuration to that of the `BlipText` used by the [base + This is the configuration class to store the configuration of a [`BlipTextModel`]. It is used to instantiate a BLIP + text model according to the specified arguments, defining the model architecture. Instantiating a configuration + with the defaults will yield a similar configuration to that of the `BlipText` used by the [base architectures](https://huggingface.co/Salesforce/blip-vqa-base). Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the @@ -294,8 +294,8 @@ def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], class BlipConfig(PretrainedConfig): r""" [`BlipConfig`] is the configuration class to store the configuration of a [`BlipModel`]. It is used to instantiate - a BLIP model according to the specified arguments, defining the text model and vision model configs. Instantiating a - configuration with the defaults will yield a similar configuration to that of the BLIP-base + a BLIP model according to the specified arguments, defining the text model and vision model configs. Instantiating + a configuration with the defaults will yield a similar configuration to that of the BLIP-base [Salesforce/blip-vqa-base](https://huggingface.co/Salesforce/blip-vqa-base) architecture. Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the From af46d12a8e36e7076dfe0b94ccc1c2d4f6091548 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 13 Dec 2022 21:09:57 +0100 Subject: [PATCH 58/96] Apply suggestions from code review Co-authored-by: NielsRogge <48327001+NielsRogge@users.noreply.github.com> --- src/transformers/models/blip/modeling_blip.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 4c6a70e84845..8a730652f6c5 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -1097,7 +1097,7 @@ def generate( ``` """ - batch_size, _ = pixel_values.shape[:2] + batch_size = pixel_values.shape[0] vision_outputs = self.vision_model( pixel_values=pixel_values, ) @@ -1131,7 +1131,7 @@ def generate( @add_start_docstrings( """ - BLIP Model fine-tuned for question answering. The model consists of a vision encoder, a text encoder as well as a + BLIP Model for visual question answering. The model consists of a vision encoder, a text encoder as well as a text decoder. The vision encoder will encode the input image, the text encoder will encode the input question together with the encoding of the image, and the text decoder will output the answer to the question. """, From 8b698fffe36e187eb29b6f6b2e80747592540503 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 20:11:46 +0000 Subject: [PATCH 59/96] make fixup --- src/transformers/utils/dummy_vision_objects.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/transformers/utils/dummy_vision_objects.py b/src/transformers/utils/dummy_vision_objects.py index bf04ca14dc59..30934cc8d238 100644 --- a/src/transformers/utils/dummy_vision_objects.py +++ b/src/transformers/utils/dummy_vision_objects.py @@ -57,13 +57,6 @@ def __init__(self, *args, **kwargs): requires_backends(self, ["vision"]) -class BlipProcessor(metaclass=DummyObject): - _backends = ["vision"] - - def __init__(self, *args, **kwargs): - requires_backends(self, ["vision"]) - - class ChineseCLIPFeatureExtractor(metaclass=DummyObject): _backends = ["vision"] From 2a3e35802fbadb7901dfd07157c645fc6653b859 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 13 Dec 2022 21:13:06 +0100 Subject: [PATCH 60/96] Apply suggestions from code review Co-authored-by: NielsRogge <48327001+NielsRogge@users.noreply.github.com> --- src/transformers/models/blip/modeling_blip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 8a730652f6c5..4996f86f704f 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -1021,7 +1021,7 @@ def forward( >>> outputs = model(**inputs) >>> image_embeds = outputs.image_embeds ```""" - batch_size, _ = pixel_values.shape[:2] + batch_size = pixel_values.shape[0] return_dict = return_dict if return_dict is not None else self.config.use_return_dict vision_outputs = self.vision_model( From f0e4f791e61e5649bd4800243b623096942dc355 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 20:15:34 +0000 Subject: [PATCH 61/96] add comments --- src/transformers/models/blip/modeling_blip.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 4996f86f704f..4bc93d33f323 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -1346,9 +1346,13 @@ def __init__(self, config: BlipConfig): self.text_encoder = BlipTextModel(config.text_config, add_pooling_layer=False) + # vision projection layer self.vision_proj = nn.Linear(config.vision_config.hidden_size, config.image_text_hidden_size) + + # text projection layer self.text_proj = nn.Linear(config.text_config.hidden_size, config.image_text_hidden_size) + # image text matching head self.itm_head = nn.Linear(config.text_config.hidden_size, 2) self.decoder_pad_token_id = config.text_config.pad_token_id From 21e4fbc7d531b8d91e0f4d7f6d35d7209d81325c Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Tue, 13 Dec 2022 21:18:01 +0100 Subject: [PATCH 62/96] Update src/transformers/models/blip/modeling_blip.py Co-authored-by: NielsRogge <48327001+NielsRogge@users.noreply.github.com> --- src/transformers/models/blip/modeling_blip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 4bc93d33f323..cb1c462f8a54 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -1193,7 +1193,7 @@ def forward( >>> outputs = model.generate(**inputs) ```""" return_dict = return_dict if return_dict is not None else self.config.use_return_dict - batch_size, _ = input_ids.shape + batch_size = input_ids.shape[0] vision_outputs = self.vision_model( pixel_values=pixel_values, From 4968a662446b834c82ef377991fc988830d54bdc Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 20:19:11 +0000 Subject: [PATCH 63/96] remove `token_type_ids` --- src/transformers/models/blip/modeling_blip.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index cb1c462f8a54..02923333c93a 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -1249,7 +1249,6 @@ def generate( input_ids: torch.LongTensor, pixel_values: torch.FloatTensor, attention_mask: Optional[torch.LongTensor] = None, - token_type_ids: Optional[torch.LongTensor] = None, **generate_kwargs ) -> torch.LongTensor: r""" @@ -1263,9 +1262,6 @@ def generate( attention_mask (*torch.LongTensor* of shape *(batch_size, sequence_length)*, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`. `1` for tokens that are NOT MASKED, `0` for MASKED tokens. - token_type_ids (*torch.LongTensor* of shape *(batch_size, sequence_length)*, *optional*): - Segment token indices to indicate first and second portions of the inputs. Not used but kept for - compatibility with the *BlipProcessor* **generate_kwargs: Additional arguments passed to the *generate* function of the decoder @@ -1372,7 +1368,6 @@ def forward( pixel_values: torch.FloatTensor, use_itm_head: Optional[bool] = True, attention_mask: Optional[torch.LongTensor] = None, - token_type_ids: Optional[torch.LongTensor] = None, output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, From a5b5cfcb63c7cad5bd65f2f2ed76112b9b275ca9 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 20:19:16 +0000 Subject: [PATCH 64/96] make fixup --- src/transformers/models/blip/modeling_blip.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 02923333c93a..baf92a9f5418 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -1131,9 +1131,9 @@ def generate( @add_start_docstrings( """ - BLIP Model for visual question answering. The model consists of a vision encoder, a text encoder as well as a - text decoder. The vision encoder will encode the input image, the text encoder will encode the input question - together with the encoding of the image, and the text decoder will output the answer to the question. + BLIP Model for visual question answering. The model consists of a vision encoder, a text encoder as well as a text + decoder. The vision encoder will encode the input image, the text encoder will encode the input question together + with the encoding of the image, and the text decoder will output the answer to the question. """, BLIP_START_DOCSTRING, ) From 57cd75b69ae071e62cf2793946695beec4373e8f Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 20:22:05 +0000 Subject: [PATCH 65/96] better name --- src/transformers/models/blip/modeling_blip.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index baf92a9f5418..7ea7a003d6af 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -1104,7 +1104,7 @@ def generate( image_embeds = vision_outputs[0] - image_atts = torch.ones(image_embeds.size()[:-1], dtype=torch.long).to(image_embeds.device) + encoder_attention_mask = torch.ones(image_embeds.size()[:-1], dtype=torch.long).to(image_embeds.device) if isinstance(input_ids, list): input_ids = torch.LongTensor(input_ids) @@ -1122,7 +1122,7 @@ def generate( pad_token_id=self.config.text_config.pad_token_id, attention_mask=attention_mask, encoder_hidden_states=image_embeds, - encoder_attention_mask=image_atts, + encoder_attention_mask=encoder_attention_mask, **generate_kwargs, ) From fbd23a40ad556d19de4d6fe5fed0bc1fa06a1668 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 20:23:11 +0000 Subject: [PATCH 66/96] replace with `image_attention_mask` --- src/transformers/models/blip/modeling_blip.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 7ea7a003d6af..d809195b4364 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -1104,7 +1104,7 @@ def generate( image_embeds = vision_outputs[0] - encoder_attention_mask = torch.ones(image_embeds.size()[:-1], dtype=torch.long).to(image_embeds.device) + image_attention_mask = torch.ones(image_embeds.size()[:-1], dtype=torch.long).to(image_embeds.device) if isinstance(input_ids, list): input_ids = torch.LongTensor(input_ids) @@ -1122,7 +1122,7 @@ def generate( pad_token_id=self.config.text_config.pad_token_id, attention_mask=attention_mask, encoder_hidden_states=image_embeds, - encoder_attention_mask=encoder_attention_mask, + encoder_attention_mask=image_attention_mask, **generate_kwargs, ) @@ -1203,13 +1203,13 @@ def forward( ) image_embeds = vision_outputs[0] - image_atts = torch.ones(image_embeds.size()[:-1], dtype=torch.long) + image_attention_mask = torch.ones(image_embeds.size()[:-1], dtype=torch.long) question_embeds = self.text_encoder( input_ids=input_ids, attention_mask=attention_mask, encoder_hidden_states=image_embeds, - encoder_attention_mask=image_atts, + encoder_attention_mask=image_attention_mask, return_dict=return_dict, ) From fc4b239c521438522b4166baa9d5decb0e121957 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 20:24:25 +0000 Subject: [PATCH 67/96] refactor --- src/transformers/models/blip/modeling_blip.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index d809195b4364..218ebf7cc8a7 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -1306,9 +1306,8 @@ def generate( question_embeds = question_outputs[0] - question_atts = torch.ones(question_embeds.size()[:-1], dtype=torch.long).to(question_embeds.device) - model_kwargs = {"encoder_hidden_states": question_embeds, "encoder_attention_mask": question_atts} - + question_attention_mask = torch.ones(question_embeds.size()[:-1], dtype=torch.long).to(question_embeds.device) + bos_ids = torch.full( (question_embeds.size(0), 1), fill_value=self.decoder_bos_token_id, device=question_embeds.device ) @@ -1317,8 +1316,9 @@ def generate( input_ids=bos_ids, eos_token_id=self.config.text_config.sep_token_id, pad_token_id=self.config.text_config.pad_token_id, + encoder_hidden_states=question_embeds, + encoder_attention_mask=question_attention_mask, **generate_kwargs, - **model_kwargs, ) return outputs From 7151c7f36636ddabc7c489f4b008d717eb990d2d Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 20:28:55 +0000 Subject: [PATCH 68/96] make fixup --- src/transformers/models/blip/modeling_blip.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 218ebf7cc8a7..e0a2ad7c2468 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -990,7 +990,7 @@ def get_input_embeddings(self) -> nn.Module: return self.vision_model.embeddings.patch_embedding @add_start_docstrings_to_model_forward(BLIP_VISION_INPUTS_DOCSTRING) - @replace_return_docstrings(output_type=BlipTextVisionModelOutput, config_class=BlipVisionConfig) + @replace_return_docstrings(output_type=BlipForConditionalGenerationModelOutput, config_class=BlipVisionConfig) def forward( self, pixel_values: torch.FloatTensor, @@ -1290,8 +1290,7 @@ def generate( image_embeds = vision_outputs[0] - image_atts = torch.ones(image_embeds.size()[:-1], dtype=torch.long).to(image_embeds.device) - model_kwargs = {"encoder_hidden_states": image_embeds, "encoder_attention_mask": image_atts} + image_attention_mask = torch.ones(image_embeds.size()[:-1], dtype=torch.long).to(image_embeds.device) if isinstance(input_ids, list): input_ids = torch.LongTensor(input_ids) @@ -1300,14 +1299,14 @@ def generate( input_ids=input_ids, attention_mask=attention_mask, encoder_hidden_states=image_embeds, - encoder_attention_mask=image_atts, + encoder_attention_mask=image_attention_mask, return_dict=False, ) question_embeds = question_outputs[0] question_attention_mask = torch.ones(question_embeds.size()[:-1], dtype=torch.long).to(question_embeds.device) - + bos_ids = torch.full( (question_embeds.size(0), 1), fill_value=self.decoder_bos_token_id, device=question_embeds.device ) From 1463f2adad5504e3c31d99724747ee8ef8b71b1f Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 20:30:27 +0000 Subject: [PATCH 69/96] better docstring --- src/transformers/models/blip/modeling_blip.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index e0a2ad7c2468..fb3c5006bcb5 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -1018,8 +1018,8 @@ def forward( >>> inputs = processor(images=image, return_tensors="pt") - >>> outputs = model(**inputs) - >>> image_embeds = outputs.image_embeds + >>> outputs = model.generate(**inputs) + >>> print(processor.decode(outputs[0], skip_special_tokens=True)) ```""" batch_size = pixel_values.shape[0] return_dict = return_dict if return_dict is not None else self.config.use_return_dict From 918b5c9f0b6b844bdc29a8db23b4b0b42d26f95d Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 20:40:18 +0000 Subject: [PATCH 70/96] replace `answer_xx` --- src/transformers/models/blip/modeling_blip.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index fb3c5006bcb5..7f5e8b54fec8 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -1165,8 +1165,8 @@ def forward( self, input_ids: torch.LongTensor, pixel_values: torch.FloatTensor, - answer_input_ids: Optional[torch.LongTensor] = None, - answer_attention_mask: Optional[torch.LongTensor] = None, + decoder_input_ids: Optional[torch.LongTensor] = None, + decoder_attention_mask: Optional[torch.LongTensor] = None, attention_mask: Optional[torch.LongTensor] = None, output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, @@ -1215,16 +1215,16 @@ def forward( question_embeds = question_embeds[0] if not return_dict else question_embeds.last_hidden_state - if answer_input_ids is None: - answer_input_ids = torch.LongTensor([self.decoder_bos_token_id]).repeat((batch_size, 1)) - answer_targets = answer_input_ids.masked_fill(answer_input_ids == self.decoder_pad_token_id, -100) + if decoder_input_ids is None: + decoder_input_ids = torch.LongTensor([self.decoder_bos_token_id]).repeat((batch_size, 1)) + labels = decoder_input_ids.masked_fill(decoder_input_ids == self.decoder_pad_token_id, -100) answer_output = self.text_decoder( - input_ids=answer_input_ids, - attention_mask=answer_attention_mask, + input_ids=decoder_input_ids, + attention_mask=decoder_attention_mask, encoder_hidden_states=question_embeds, encoder_attention_mask=attention_mask, - labels=answer_targets, + labels=labels, return_dict=return_dict, reduction="none", ) From 7a943b3792c41fd569cd063d76a64dd7d5d0a8f4 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 21:49:02 +0000 Subject: [PATCH 71/96] remove ununsed args --- src/transformers/models/blip/modeling_blip.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 7f5e8b54fec8..96427f0cc3e8 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -775,8 +775,6 @@ def get_text_features( input_ids: Optional[torch.Tensor] = None, attention_mask: Optional[torch.Tensor] = None, position_ids: Optional[torch.Tensor] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> torch.FloatTensor: r""" @@ -795,19 +793,12 @@ def get_text_features( >>> inputs = processor(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") >>> text_features = model.get_text_features(**inputs) ```""" - # Use BLIP model's config for some fields (if specified) instead of those of vision & text components. - output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions - output_hidden_states = ( - output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states - ) return_dict = return_dict if return_dict is not None else self.config.use_return_dict text_outputs = self.text_model( input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, return_dict=return_dict, ) @@ -820,8 +811,6 @@ def get_text_features( def get_image_features( self, pixel_values: Optional[torch.FloatTensor] = None, - output_attentions: Optional[bool] = None, - output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None, ) -> torch.FloatTensor: r""" @@ -846,17 +835,10 @@ def get_image_features( >>> image_features = model.get_image_features(**inputs) ```""" - # Use BLIP model's config for some fields (if specified) instead of those of vision & text components. - output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions - output_hidden_states = ( - output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states - ) return_dict = return_dict if return_dict is not None else self.config.use_return_dict vision_outputs = self.vision_model( pixel_values=pixel_values, - output_attentions=output_attentions, - output_hidden_states=output_hidden_states, return_dict=return_dict, ) From 923aa1cfa538a2a4af67846646e34a0cdf6a3651 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 21:53:20 +0000 Subject: [PATCH 72/96] add `labels` --- src/transformers/models/blip/modeling_blip.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 96427f0cc3e8..13b8161bbd18 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -980,6 +980,7 @@ def forward( attention_mask: Optional[torch.LongTensor] = None, output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, + labels: Optional[torch.LongTensor] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BlipForConditionalGenerationModelOutput]: r""" @@ -1018,13 +1019,14 @@ def forward( if input_ids is None: input_ids = torch.LongTensor([[self.decoder_input_ids] * batch_size]).to(image_embeds.device) - decoder_targets = input_ids.masked_fill(input_ids == self.decoder_pad_token_id, -100) + if labels is None: + labels = input_ids.masked_fill(input_ids == self.decoder_pad_token_id, -100) outputs = self.text_decoder( input_ids=input_ids, attention_mask=attention_mask, encoder_hidden_states=image_embeds, - labels=decoder_targets, + labels=labels, return_dict=return_dict, ) From 6757711ac830024dff22589d415fb40fb392cd29 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 21:55:17 +0000 Subject: [PATCH 73/96] add `labels` --- src/transformers/models/blip/modeling_blip.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 13b8161bbd18..b2bfdebbd510 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -1154,6 +1154,7 @@ def forward( attention_mask: Optional[torch.LongTensor] = None, output_attentions: Optional[bool] = None, output_hidden_states: Optional[bool] = None, + labels: Optional[torch.LongTensor] = None, return_dict: Optional[bool] = None, ) -> Union[Tuple, BlipTextVisionModelOutput]: r""" @@ -1201,7 +1202,9 @@ def forward( if decoder_input_ids is None: decoder_input_ids = torch.LongTensor([self.decoder_bos_token_id]).repeat((batch_size, 1)) - labels = decoder_input_ids.masked_fill(decoder_input_ids == self.decoder_pad_token_id, -100) + + if labels is None: + labels = decoder_input_ids.masked_fill(decoder_input_ids == self.decoder_pad_token_id, -100) answer_output = self.text_decoder( input_ids=decoder_input_ids, From d430a58b083036f9baf1a5516cf5d2f464921094 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 23:03:51 +0000 Subject: [PATCH 74/96] fix processing tests --- .../models/blip/image_processing_blip.py | 74 ++++- src/transformers/models/blip/modeling_blip.py | 2 +- .../models/blip/processing_blip.py | 12 +- .../models/blip/test_image_processing_blip.py | 299 ++++++++++++++++++ tests/models/blip/test_processor_blip.py | 155 +++++++++ 5 files changed, 535 insertions(+), 7 deletions(-) create mode 100644 tests/models/blip/test_image_processing_blip.py create mode 100644 tests/models/blip/test_processor_blip.py diff --git a/src/transformers/models/blip/image_processing_blip.py b/src/transformers/models/blip/image_processing_blip.py index 9586b937f108..fa7b6952e038 100644 --- a/src/transformers/models/blip/image_processing_blip.py +++ b/src/transformers/models/blip/image_processing_blip.py @@ -23,7 +23,7 @@ from transformers.utils.generic import TensorType from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict -from ...image_transforms import normalize, rescale, resize, to_channel_dimension_format +from ...image_transforms import center_crop, normalize, rescale, resize, to_channel_dimension_format from ...image_utils import ( IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD, @@ -46,6 +46,21 @@ logger = logging.get_logger(__name__) +# Copied from transformers.models.clip.image_processing_clip.convert_to_rgb +def convert_to_rgb(image: Union[Any, PIL.Image.Image]) -> Union[Any, PIL.Image.Image]: + """ + Converts `PIL.Image.Image` to RGB format. Images in other formats are returned as is. + + Args: + image (`PIL.Image.Image`): + The image to convert. + """ + if not isinstance(image, PIL.Image.Image): + return image + + return image.convert("RGB") + + # Copied from transformers.models.vilt.image_processing_vilt.max_across_indices def max_across_indices(values: Iterable[Any]) -> List[Any]: """ @@ -175,6 +190,9 @@ class BlipImageProcessor(BaseImageProcessor): do_pad (`bool`, *optional*, defaults to `True`): Whether to pad the image to the `(max_height, max_width)` of the images in the batch. Can be overridden by the `do_pad` parameter in the `preprocess` method. + do_convert_rgb (`bool`, *optional*, defaults to `True`): + Standard deviation to use if normalizing the image. This is a float or list of floats the length of the + number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method. """ model_input_names = ["pixel_values"] @@ -185,22 +203,29 @@ def __init__( size: Dict[str, int] = None, size_divisor: int = 32, resample: PILImageResampling = PILImageResampling.BICUBIC, + do_center_crop: bool = False, + crop_size: Dict[str, int] = None, do_rescale: bool = True, rescale_factor: Union[int, float] = 1 / 255, do_normalize: bool = True, image_mean: Optional[Union[float, List[float]]] = None, image_std: Optional[Union[float, List[float]]] = None, - do_pad: bool = True, + do_pad: bool = False, + do_convert_rgb: bool = True, **kwargs ) -> None: super().__init__(**kwargs) size = size if size is not None else 384 size = get_size_dict(size, default_to_square=True) + crop_size = crop_size if crop_size is not None else {"height": 224, "width": 224} + crop_size = get_size_dict(crop_size, default_to_square=True, param_name="crop_size") self.do_resize = do_resize self.size = size self.size_divisor = size_divisor + self.do_center_crop = do_center_crop + self.crop_size = crop_size self.resample = resample self.do_rescale = do_rescale self.rescale_factor = rescale_factor @@ -208,6 +233,31 @@ def __init__( self.image_mean = image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN self.image_std = image_std if image_std is not None else IMAGENET_STANDARD_STD self.do_pad = do_pad + self.do_convert_rgb = do_convert_rgb + + def center_crop( + self, + image: np.ndarray, + size: Dict[str, int], + data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs + ) -> np.ndarray: + """ + Center crop an image. If the image is too small to be cropped to the size given, it will be padded (so the + returned result will always be of size `size`). + + Args: + image (`np.ndarray`): + Image to center crop. + size (`Dict[str, int]`): + Size of the output image in the form of a dictionary with keys `height` and `width`. + data_format (`str` or `ChannelDimension`, *optional*): + The channel dimension format of the image. If not provided, it will be the same as the input image. + """ + size = get_size_dict(size) + if "height" not in size or "width" not in size: + raise ValueError(f"The `size` parameter must contain the keys (height, width). Got {size.keys()}") + return center_crop(image, size=(size["height"], size["width"]), data_format=data_format, **kwargs) def resize( self, @@ -352,12 +402,15 @@ def preprocess( size_divisor: Optional[int] = None, resample: PILImageResampling = None, do_rescale: Optional[bool] = None, + do_center_crop: bool = None, + crop_size: int = None, rescale_factor: Optional[float] = None, do_normalize: Optional[bool] = None, image_mean: Optional[Union[float, List[float]]] = None, image_std: Optional[Union[float, List[float]]] = None, do_pad: Optional[bool] = None, return_tensors: Optional[Union[str, TensorType]] = None, + do_convert_rgb: bool = None, data_format: ChannelDimension = ChannelDimension.FIRST, **kwargs, ) -> PIL.Image.Image: @@ -378,6 +431,10 @@ def preprocess( The image is resized to a size that is a multiple of this value. resample (`PILImageResampling`, *optional*, defaults to `self.resample`): Resampling filter to use if resizing the image. Only has an effect if `do_resize` is set to `True`. + do_center_crop (`bool`, *optional*, defaults to `self.do_center_crop`): + Whether to center crop the image. + crop_size (`Dict[str, int]`, *optional*, defaults to `self.crop_size`): + Size of the center crop. Only has an effect if `do_center_crop` is set to `True`. do_rescale (`bool`, *optional*, defaults to `self.do_rescale`): Whether to rescale the image values between [0 - 1]. rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`): @@ -391,6 +448,8 @@ def preprocess( do_pad (`bool`, *optional*, defaults to `self.do_pad`): Whether to pad the image to the (max_height, max_width) in the batch. If `True`, a pixel mask is also created and returned. + do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`): + Whether to convert the image to RGB. return_tensors (`str` or `TensorType`, *optional*): The type of tensors to return. Can be one of: - Unset: Return a list of `np.ndarray`. @@ -406,12 +465,16 @@ def preprocess( do_resize = do_resize if do_resize is not None else self.do_resize size_divisor = size_divisor if size_divisor is not None else self.size_divisor resample = resample if resample is not None else self.resample + do_center_crop = do_center_crop if do_center_crop is not None else self.do_center_crop + crop_size = crop_size if crop_size is not None else self.crop_size + crop_size = get_size_dict(crop_size, param_name="crop_size", default_to_square=True) do_rescale = do_rescale if do_rescale is not None else self.do_rescale rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor do_normalize = do_normalize if do_normalize is not None else self.do_normalize image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std do_pad = do_pad if do_pad is not None else self.do_pad + do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb size = size if size is not None else self.size size = get_size_dict(size, default_to_square=False) @@ -434,12 +497,19 @@ def preprocess( if do_normalize and (image_mean is None or image_std is None): raise ValueError("Image mean and std must be specified if do_normalize is True.") + # PIL RGBA images are converted to RGB + if do_convert_rgb: + images = [convert_to_rgb(image) for image in images] + # All transformations expect numpy arrays. images = [to_numpy_array(image) for image in images] if do_resize: images = [self.resize(image=image, size=size, resample=resample) for image in images] + if do_center_crop: + images = [self.center_crop(image=image, size=crop_size) for image in images] + if do_rescale: images = [self.rescale(image=image, scale=rescale_factor) for image in images] diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index b2bfdebbd510..d24e712882fd 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -1202,7 +1202,7 @@ def forward( if decoder_input_ids is None: decoder_input_ids = torch.LongTensor([self.decoder_bos_token_id]).repeat((batch_size, 1)) - + if labels is None: labels = decoder_input_ids.masked_fill(decoder_input_ids == self.decoder_pad_token_id, -100) diff --git a/src/transformers/models/blip/processing_blip.py b/src/transformers/models/blip/processing_blip.py index b609961de63e..a65e90bb4d86 100644 --- a/src/transformers/models/blip/processing_blip.py +++ b/src/transformers/models/blip/processing_blip.py @@ -40,12 +40,13 @@ class BlipProcessor(ProcessorMixin): tokenizer_class = ("BertTokenizer", "BertTokenizerFast") def __init__(self, feature_extractor, tokenizer): + tokenizer.return_token_type_ids = False super().__init__(feature_extractor, tokenizer) self.current_processor = self.feature_extractor def __call__( self, - images, + images=None, text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None, add_special_tokens: bool = True, padding: Union[bool, str, PaddingStrategy] = False, @@ -57,6 +58,7 @@ def __call__( return_overflowing_tokens: bool = False, return_special_tokens_mask: bool = False, return_offsets_mapping: bool = False, + return_token_type_ids: bool = False, return_length: bool = False, verbose: bool = True, return_tensors: Optional[Union[str, TensorType]] = None, @@ -68,10 +70,11 @@ def __call__( Please refer to the docstring of the above two methods for more information. """ + if images is None and text is None: + raise ValueError("You have to specify either images or text.") + # Get only text if images is None: - if text is None: - raise ValueError("You have to specify either images or text.") self.current_processor = self.tokenizer text_encoding = self.tokenizer( @@ -86,6 +89,7 @@ def __call__( return_overflowing_tokens=return_overflowing_tokens, return_special_tokens_mask=return_special_tokens_mask, return_offsets_mapping=return_offsets_mapping, + return_token_type_ids=return_token_type_ids, return_length=return_length, verbose=verbose, return_tensors=return_tensors, @@ -110,12 +114,12 @@ def __call__( return_overflowing_tokens=return_overflowing_tokens, return_special_tokens_mask=return_special_tokens_mask, return_offsets_mapping=return_offsets_mapping, + return_token_type_ids=return_token_type_ids, return_length=return_length, verbose=verbose, return_tensors=return_tensors, **kwargs, ) - _ = text_encoding.pop("token_type_ids", None) else: text_encoding = None diff --git a/tests/models/blip/test_image_processing_blip.py b/tests/models/blip/test_image_processing_blip.py new file mode 100644 index 000000000000..0ff5f21c2e80 --- /dev/null +++ b/tests/models/blip/test_image_processing_blip.py @@ -0,0 +1,299 @@ +# coding=utf-8 +# Copyright 2022 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import unittest + +import numpy as np + +from transformers.testing_utils import require_torch, require_vision +from transformers.utils import is_torch_available, is_vision_available + +from ...test_feature_extraction_common import FeatureExtractionSavingTestMixin + + +if is_torch_available(): + import torch + +if is_vision_available(): + from PIL import Image + + from transformers import BlipImageProcessor + + +class BlipImageProcessingTester(unittest.TestCase): + def __init__( + self, + parent, + batch_size=7, + num_channels=3, + image_size=18, + min_resolution=30, + max_resolution=400, + do_resize=True, + size=None, + do_center_crop=True, + crop_size=None, + do_normalize=True, + do_pad=False, + image_mean=[0.48145466, 0.4578275, 0.40821073], + image_std=[0.26862954, 0.26130258, 0.27577711], + do_convert_rgb=True, + ): + size = size if size is not None else {"height": 20, "width": 20} + crop_size = crop_size if crop_size is not None else {"height": 18, "width": 18} + self.parent = parent + self.batch_size = batch_size + self.num_channels = num_channels + self.image_size = image_size + self.min_resolution = min_resolution + self.max_resolution = max_resolution + self.do_resize = do_resize + self.size = size + self.do_center_crop = do_center_crop + self.crop_size = crop_size + self.do_normalize = do_normalize + self.image_mean = image_mean + self.image_std = image_std + self.do_pad = do_pad + self.do_convert_rgb = do_convert_rgb + + def prepare_feat_extract_dict(self): + return { + "do_resize": self.do_resize, + "size": self.size, + "do_center_crop": self.do_center_crop, + "crop_size": self.crop_size, + "do_normalize": self.do_normalize, + "image_mean": self.image_mean, + "image_std": self.image_std, + "do_convert_rgb": self.do_convert_rgb, + "do_pad": self.do_pad, + } + + def prepare_inputs(self, equal_resolution=False, numpify=False, torchify=False): + """This function prepares a list of PIL images, or a list of numpy arrays if one specifies numpify=True, + or a list of PyTorch tensors if one specifies torchify=True. + """ + + assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time" + + if equal_resolution: + image_inputs = [] + for i in range(self.batch_size): + image_inputs.append( + np.random.randint( + 255, size=(self.num_channels, self.max_resolution, self.max_resolution), dtype=np.uint8 + ) + ) + else: + image_inputs = [] + for i in range(self.batch_size): + width, height = np.random.choice(np.arange(self.min_resolution, self.max_resolution), 2) + image_inputs.append(np.random.randint(255, size=(self.num_channels, width, height), dtype=np.uint8)) + + if not numpify and not torchify: + # PIL expects the channel dimension as last dimension + image_inputs = [Image.fromarray(np.moveaxis(x, 0, -1)) for x in image_inputs] + + if torchify: + image_inputs = [torch.from_numpy(x) for x in image_inputs] + + return image_inputs + + +@require_torch +@require_vision +class BlipImageProcessingTest(FeatureExtractionSavingTestMixin, unittest.TestCase): + + feature_extraction_class = BlipImageProcessor if is_vision_available() else None + + def setUp(self): + self.feature_extract_tester = BlipImageProcessingTester(self) + + @property + def feat_extract_dict(self): + return self.feature_extract_tester.prepare_feat_extract_dict() + + def test_feat_extract_properties(self): + feature_extractor = self.feature_extraction_class(**self.feat_extract_dict) + self.assertTrue(hasattr(feature_extractor, "do_resize")) + self.assertTrue(hasattr(feature_extractor, "size")) + self.assertTrue(hasattr(feature_extractor, "do_center_crop")) + self.assertTrue(hasattr(feature_extractor, "center_crop")) + self.assertTrue(hasattr(feature_extractor, "do_normalize")) + self.assertTrue(hasattr(feature_extractor, "image_mean")) + self.assertTrue(hasattr(feature_extractor, "image_std")) + self.assertTrue(hasattr(feature_extractor, "do_convert_rgb")) + + def test_batch_feature(self): + pass + + def test_call_pil(self): + # Initialize feature_extractor + feature_extractor = self.feature_extraction_class(**self.feat_extract_dict) + # create random PIL images + image_inputs = self.feature_extract_tester.prepare_inputs(equal_resolution=False) + for image in image_inputs: + self.assertIsInstance(image, Image.Image) + + # Test not batched input + encoded_images = feature_extractor(image_inputs[0], return_tensors="pt").pixel_values + self.assertEqual( + encoded_images.shape, + ( + 1, + self.feature_extract_tester.num_channels, + self.feature_extract_tester.crop_size["height"], + self.feature_extract_tester.crop_size["width"], + ), + ) + + # Test batched + encoded_images = feature_extractor(image_inputs, return_tensors="pt").pixel_values + self.assertEqual( + encoded_images.shape, + ( + self.feature_extract_tester.batch_size, + self.feature_extract_tester.num_channels, + self.feature_extract_tester.crop_size["height"], + self.feature_extract_tester.crop_size["width"], + ), + ) + + def test_call_numpy(self): + # Initialize feature_extractor + feature_extractor = self.feature_extraction_class(**self.feat_extract_dict) + # create random numpy tensors + image_inputs = self.feature_extract_tester.prepare_inputs(equal_resolution=False, numpify=True) + for image in image_inputs: + self.assertIsInstance(image, np.ndarray) + + # Test not batched input + encoded_images = feature_extractor(image_inputs[0], return_tensors="pt").pixel_values + self.assertEqual( + encoded_images.shape, + ( + 1, + self.feature_extract_tester.num_channels, + self.feature_extract_tester.crop_size["height"], + self.feature_extract_tester.crop_size["width"], + ), + ) + + # Test batched + encoded_images = feature_extractor(image_inputs, return_tensors="pt").pixel_values + self.assertEqual( + encoded_images.shape, + ( + self.feature_extract_tester.batch_size, + self.feature_extract_tester.num_channels, + self.feature_extract_tester.crop_size["height"], + self.feature_extract_tester.crop_size["width"], + ), + ) + + def test_call_pytorch(self): + # Initialize feature_extractor + feature_extractor = self.feature_extraction_class(**self.feat_extract_dict) + # create random PyTorch tensors + image_inputs = self.feature_extract_tester.prepare_inputs(equal_resolution=False, torchify=True) + for image in image_inputs: + self.assertIsInstance(image, torch.Tensor) + + # Test not batched input + encoded_images = feature_extractor(image_inputs[0], return_tensors="pt").pixel_values + self.assertEqual( + encoded_images.shape, + ( + 1, + self.feature_extract_tester.num_channels, + self.feature_extract_tester.crop_size["height"], + self.feature_extract_tester.crop_size["width"], + ), + ) + + # Test batched + encoded_images = feature_extractor(image_inputs, return_tensors="pt").pixel_values + self.assertEqual( + encoded_images.shape, + ( + self.feature_extract_tester.batch_size, + self.feature_extract_tester.num_channels, + self.feature_extract_tester.crop_size["height"], + self.feature_extract_tester.crop_size["width"], + ), + ) + + +@require_torch +@require_vision +class BlipImageProcessingTestFourChannels(FeatureExtractionSavingTestMixin, unittest.TestCase): + + feature_extraction_class = BlipImageProcessor if is_vision_available() else None + + def setUp(self): + self.feature_extract_tester = BlipImageProcessingTester(self, num_channels=4) + self.expected_encoded_image_num_channels = 3 + + @property + def feat_extract_dict(self): + return self.feature_extract_tester.prepare_feat_extract_dict() + + def test_feat_extract_properties(self): + feature_extractor = self.feature_extraction_class(**self.feat_extract_dict) + self.assertTrue(hasattr(feature_extractor, "do_resize")) + self.assertTrue(hasattr(feature_extractor, "size")) + self.assertTrue(hasattr(feature_extractor, "do_center_crop")) + self.assertTrue(hasattr(feature_extractor, "center_crop")) + self.assertTrue(hasattr(feature_extractor, "do_normalize")) + self.assertTrue(hasattr(feature_extractor, "image_mean")) + self.assertTrue(hasattr(feature_extractor, "image_std")) + self.assertTrue(hasattr(feature_extractor, "do_convert_rgb")) + + def test_batch_feature(self): + pass + + def test_call_pil_four_channels(self): + # Initialize feature_extractor + feature_extractor = self.feature_extraction_class(**self.feat_extract_dict) + # create random PIL images + image_inputs = self.feature_extract_tester.prepare_inputs(equal_resolution=False) + for image in image_inputs: + self.assertIsInstance(image, Image.Image) + + # Test not batched input + encoded_images = feature_extractor(image_inputs[0], return_tensors="pt").pixel_values + self.assertEqual( + encoded_images.shape, + ( + 1, + self.expected_encoded_image_num_channels, + self.feature_extract_tester.crop_size["height"], + self.feature_extract_tester.crop_size["width"], + ), + ) + + # Test batched + encoded_images = feature_extractor(image_inputs, return_tensors="pt").pixel_values + self.assertEqual( + encoded_images.shape, + ( + self.feature_extract_tester.batch_size, + self.expected_encoded_image_num_channels, + self.feature_extract_tester.crop_size["height"], + self.feature_extract_tester.crop_size["width"], + ), + ) \ No newline at end of file diff --git a/tests/models/blip/test_processor_blip.py b/tests/models/blip/test_processor_blip.py new file mode 100644 index 000000000000..ffd98a271825 --- /dev/null +++ b/tests/models/blip/test_processor_blip.py @@ -0,0 +1,155 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import shutil +import tempfile +import unittest + +import numpy as np +import pytest + +from transformers.testing_utils import require_vision +from transformers.utils import is_vision_available + + +if is_vision_available(): + from PIL import Image + + from transformers import BlipImageProcessor, BlipProcessor, AutoProcessor, BertTokenizer, PreTrainedTokenizerFast + + +@require_vision +class BlipProcessorTest(unittest.TestCase): + def setUp(self): + self.tmpdirname = tempfile.mkdtemp() + + image_processor = BlipImageProcessor() + tokenizer = BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-BertModel") + + processor = BlipProcessor(image_processor, tokenizer) + + processor.save_pretrained(self.tmpdirname) + + def get_tokenizer(self, **kwargs): + return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer + + + def get_feature_extractor(self, **kwargs): + return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).feature_extractor + + def tearDown(self): + shutil.rmtree(self.tmpdirname) + + def prepare_image_inputs(self): + """This function prepares a list of PIL images, or a list of numpy arrays if one specifies numpify=True, + or a list of PyTorch tensors if one specifies torchify=True. + """ + + image_inputs = [np.random.randint(255, size=(3, 30, 400), dtype=np.uint8)] + + image_inputs = [Image.fromarray(np.moveaxis(x, 0, -1)) for x in image_inputs] + + return image_inputs + + def test_save_load_pretrained_additional_features(self): + processor = BlipProcessor(tokenizer=self.get_tokenizer(), feature_extractor=self.get_feature_extractor()) + processor.save_pretrained(self.tmpdirname) + + tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)") + feature_extractor_add_kwargs = self.get_feature_extractor(do_normalize=False, padding_value=1.0) + + processor = BlipProcessor.from_pretrained( + self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0 + ) + + self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab()) + self.assertIsInstance(processor.tokenizer, PreTrainedTokenizerFast) + + self.assertEqual(processor.feature_extractor.to_json_string(), feature_extractor_add_kwargs.to_json_string()) + self.assertIsInstance(processor.feature_extractor, BlipImageProcessor) + + def test_feature_extractor(self): + feature_extractor = self.get_feature_extractor() + tokenizer = self.get_tokenizer() + + processor = BlipProcessor(tokenizer=tokenizer, feature_extractor=feature_extractor) + + image_input = self.prepare_image_inputs() + + input_feat_extract = feature_extractor(image_input, return_tensors="np") + input_processor = processor(images=image_input, return_tensors="np") + + for key in input_feat_extract.keys(): + self.assertAlmostEqual(input_feat_extract[key].sum(), input_processor[key].sum(), delta=1e-2) + + def test_tokenizer(self): + feature_extractor = self.get_feature_extractor() + tokenizer = self.get_tokenizer() + + processor = BlipProcessor(tokenizer=tokenizer, feature_extractor=feature_extractor) + + input_str = "lower newer" + + encoded_processor = processor(text=input_str) + + encoded_tok = tokenizer(input_str, return_token_type_ids=False) + + for key in encoded_tok.keys(): + self.assertListEqual(encoded_tok[key], encoded_processor[key]) + + def test_processor(self): + feature_extractor = self.get_feature_extractor() + tokenizer = self.get_tokenizer() + + processor = BlipProcessor(tokenizer=tokenizer, feature_extractor=feature_extractor) + + input_str = "lower newer" + image_input = self.prepare_image_inputs() + + inputs = processor(text=input_str, images=image_input) + + self.assertListEqual(list(inputs.keys()), ["pixel_values", "input_ids", "attention_mask"]) + + # test if it raises when no input is passed + with pytest.raises(ValueError): + processor() + + def test_tokenizer_decode(self): + feature_extractor = self.get_feature_extractor() + tokenizer = self.get_tokenizer() + + processor = BlipProcessor(tokenizer=tokenizer, feature_extractor=feature_extractor) + + predicted_ids = [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]] + + decoded_processor = processor.batch_decode(predicted_ids) + decoded_tok = tokenizer.batch_decode(predicted_ids) + + self.assertListEqual(decoded_tok, decoded_processor) + + def test_model_input_names(self): + feature_extractor = self.get_feature_extractor() + tokenizer = self.get_tokenizer() + + processor = BlipProcessor(tokenizer=tokenizer, feature_extractor=feature_extractor) + + input_str = "lower newer" + image_input = self.prepare_image_inputs() + + inputs = processor(text=input_str, images=image_input) + + # For now the processor supports only ['pixel_values', 'input_ids', 'attention_mask'] + self.assertListEqual(list(inputs.keys()), ['pixel_values', 'input_ids', 'attention_mask']) From 92c69c3ef98e181dda8c6adc569f0e1936ce89c7 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Tue, 13 Dec 2022 23:22:35 +0000 Subject: [PATCH 75/96] make fixup --- src/transformers/models/blip/modeling_blip.py | 8 +++++--- tests/models/blip/test_modeling_blip.py | 8 ++++---- tests/models/blip/test_processor_blip.py | 10 +++------- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index d24e712882fd..4c77b62d4992 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -1093,9 +1093,11 @@ def generate( if isinstance(input_ids, list): input_ids = torch.LongTensor(input_ids) elif input_ids is None: - input_ids = torch.LongTensor( - [[self.decoder_input_ids, self.config.text_config.eos_token_id] * batch_size] - ).to(image_embeds.device) + input_ids = ( + torch.LongTensor([[self.decoder_input_ids, self.config.text_config.eos_token_id]]) + .repeat(batch_size, 1) + .to(image_embeds.device) + ) input_ids[:, 0] = self.config.text_config.bos_token_id attention_mask = attention_mask[:, :-1] if attention_mask is not None else None diff --git a/tests/models/blip/test_modeling_blip.py b/tests/models/blip/test_modeling_blip.py index dedcfce5a314..58e081d2638a 100644 --- a/tests/models/blip/test_modeling_blip.py +++ b/tests/models/blip/test_modeling_blip.py @@ -802,8 +802,8 @@ def test_inference_image_captioning(self): ) def test_inference_vqa(self): - model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base").to(torch_device) - processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base") + model = BlipForQuestionAnswering.from_pretrained("ybelkada/blip-vqa-base").to(torch_device) + processor = BlipProcessor.from_pretrained("ybelkada/blip-vqa-base") image = prepare_img() text = "how many dogs are in the picture?" @@ -815,8 +815,8 @@ def test_inference_vqa(self): self.assertEqual(out[0].tolist(), [30522, 1015, 102]) def test_inference_itm(self): - model = BlipForImageTextRetrieval.from_pretrained("Salesforce/blip-itm-base").to(torch_device) - processor = BlipProcessor.from_pretrained("Salesforce/blip-itm-base") + model = BlipForImageTextRetrieval.from_pretrained("ybelkada/blip-itm-base").to(torch_device) + processor = BlipProcessor.from_pretrained("ybelkada/blip-itm-base") image = prepare_img() text = "A woman and her dog sitting in a beach" diff --git a/tests/models/blip/test_processor_blip.py b/tests/models/blip/test_processor_blip.py index ffd98a271825..15f4ea5251dc 100644 --- a/tests/models/blip/test_processor_blip.py +++ b/tests/models/blip/test_processor_blip.py @@ -11,9 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -import json -import os import shutil import tempfile import unittest @@ -28,7 +25,7 @@ if is_vision_available(): from PIL import Image - from transformers import BlipImageProcessor, BlipProcessor, AutoProcessor, BertTokenizer, PreTrainedTokenizerFast + from transformers import AutoProcessor, BertTokenizer, BlipImageProcessor, BlipProcessor, PreTrainedTokenizerFast @require_vision @@ -40,13 +37,12 @@ def setUp(self): tokenizer = BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-BertModel") processor = BlipProcessor(image_processor, tokenizer) - + processor.save_pretrained(self.tmpdirname) def get_tokenizer(self, **kwargs): return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer - def get_feature_extractor(self, **kwargs): return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).feature_extractor @@ -152,4 +148,4 @@ def test_model_input_names(self): inputs = processor(text=input_str, images=image_input) # For now the processor supports only ['pixel_values', 'input_ids', 'attention_mask'] - self.assertListEqual(list(inputs.keys()), ['pixel_values', 'input_ids', 'attention_mask']) + self.assertListEqual(list(inputs.keys()), ["pixel_values", "input_ids", "attention_mask"]) From 519ca815754ca587415e6e8f3d4e28b498ea73df Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Wed, 14 Dec 2022 08:05:23 +0000 Subject: [PATCH 76/96] make fixup --- tests/models/blip/test_image_processing_blip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/models/blip/test_image_processing_blip.py b/tests/models/blip/test_image_processing_blip.py index 0ff5f21c2e80..f8edad9a237a 100644 --- a/tests/models/blip/test_image_processing_blip.py +++ b/tests/models/blip/test_image_processing_blip.py @@ -296,4 +296,4 @@ def test_call_pil_four_channels(self): self.feature_extract_tester.crop_size["height"], self.feature_extract_tester.crop_size["width"], ), - ) \ No newline at end of file + ) From 480b94f60adfbdf0f875d02e5af1c9233d2f8fb8 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Wed, 14 Dec 2022 08:05:47 +0000 Subject: [PATCH 77/96] put correct repo --- tests/models/blip/test_modeling_blip.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/models/blip/test_modeling_blip.py b/tests/models/blip/test_modeling_blip.py index 58e081d2638a..af17cbcc3c92 100644 --- a/tests/models/blip/test_modeling_blip.py +++ b/tests/models/blip/test_modeling_blip.py @@ -777,8 +777,8 @@ def prepare_img(): @slow class BlipModelIntegrationTest(unittest.TestCase): def test_inference_image_captioning(self): - model = BlipForConditionalGeneration.from_pretrained("ybelkada/blip-image-captioning-base").to(torch_device) - processor = BlipProcessor.from_pretrained("ybelkada/blip-image-captioning-base") + model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base").to(torch_device) + processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") image = prepare_img() # image only @@ -802,8 +802,8 @@ def test_inference_image_captioning(self): ) def test_inference_vqa(self): - model = BlipForQuestionAnswering.from_pretrained("ybelkada/blip-vqa-base").to(torch_device) - processor = BlipProcessor.from_pretrained("ybelkada/blip-vqa-base") + model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base").to(torch_device) + processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base") image = prepare_img() text = "how many dogs are in the picture?" @@ -815,8 +815,8 @@ def test_inference_vqa(self): self.assertEqual(out[0].tolist(), [30522, 1015, 102]) def test_inference_itm(self): - model = BlipForImageTextRetrieval.from_pretrained("ybelkada/blip-itm-base").to(torch_device) - processor = BlipProcessor.from_pretrained("ybelkada/blip-itm-base") + model = BlipForImageTextRetrieval.from_pretrained("Salesforce/blip-itm-base").to(torch_device) + processor = BlipProcessor.from_pretrained("Salesforce/blip-itm-base") image = prepare_img() text = "A woman and her dog sitting in a beach" From 403d92a6540be26293b01fa19e5cccd5cffe12bc Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Wed, 14 Dec 2022 09:05:52 +0000 Subject: [PATCH 78/96] remove `pad` --- .../models/blip/image_processing_blip.py | 118 +----------------- 1 file changed, 1 insertion(+), 117 deletions(-) diff --git a/src/transformers/models/blip/image_processing_blip.py b/src/transformers/models/blip/image_processing_blip.py index fa7b6952e038..bae5ee61dd3d 100644 --- a/src/transformers/models/blip/image_processing_blip.py +++ b/src/transformers/models/blip/image_processing_blip.py @@ -69,47 +69,6 @@ def max_across_indices(values: Iterable[Any]) -> List[Any]: return [max(values_i) for values_i in zip(*values)] -# Copied from transformers.models.vilt.image_processing_vilt.pad -def pad( - image: np.ndarray, - output_size: Tuple[int, int], - input_channel_dimension: Optional[ChannelDimension] = None, - data_format: Optional[ChannelDimension] = None, -) -> np.ndarray: - """ - Pad the bottom and right of the image with zeros to the output size. - - Args: - image (`np.ndarray`): - Image to pad. - output_size (`Tuple[int, int]`): - Output size of the image. - input_channel_dimension (`ChannelDimension`, *optional*): - The channel dimension format of the image. If not provided, it will be inferred from the input image. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format of the image. If not provided, it will be the same as the input image. - """ - if input_channel_dimension is None: - input_channel_dimension = infer_channel_dimension_format(image) - - output_height, output_width = output_size - input_height, input_width = get_image_size(image) - pad_bottom = output_height - input_height - pad_right = output_width - input_width - - if input_channel_dimension == ChannelDimension.FIRST: - padded_image = np.pad(image, [(0, 0), (0, pad_bottom), (0, pad_right)], mode="constant", constant_values=0) - elif input_channel_dimension == ChannelDimension.LAST: - padded_image = np.pad(image, [(0, pad_bottom), (0, pad_right), (0, 0)], mode="constant", constant_values=0) - else: - raise ValueError(f"Invalid channel dimension format: {input_channel_dimension}") - - if data_format is not None: - padded_image = to_channel_dimension_format(padded_image, data_format) - - return padded_image - - # Copied from transformers.models.vilt.image_processing_vilt.get_max_dimensions def get_max_dimensions(images: List[np.ndarray]) -> List[int]: """ @@ -187,9 +146,6 @@ class BlipImageProcessor(BaseImageProcessor): Standard deviation to use if normalizing the image. This is a float or list of floats the length of the number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method. Can be overridden by the `image_std` parameter in the `preprocess` method. - do_pad (`bool`, *optional*, defaults to `True`): - Whether to pad the image to the `(max_height, max_width)` of the images in the batch. Can be overridden by - the `do_pad` parameter in the `preprocess` method. do_convert_rgb (`bool`, *optional*, defaults to `True`): Standard deviation to use if normalizing the image. This is a float or list of floats the length of the number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method. @@ -210,7 +166,6 @@ def __init__( do_normalize: bool = True, image_mean: Optional[Union[float, List[float]]] = None, image_std: Optional[Union[float, List[float]]] = None, - do_pad: bool = False, do_convert_rgb: bool = True, **kwargs ) -> None: @@ -232,7 +187,6 @@ def __init__( self.do_normalize = do_normalize self.image_mean = image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN self.image_std = image_std if image_std is not None else IMAGENET_STANDARD_STD - self.do_pad = do_pad self.do_convert_rgb = do_convert_rgb def center_crop( @@ -331,68 +285,6 @@ def normalize( """ return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) - def pad( - self, - images: List[np.ndarray], - return_tensors: Optional[Union[str, TensorType]] = None, - data_format: Optional[ChannelDimension] = None, - ) -> BatchFeature: - """ - Pads a batch of images with zeros to the size of largest height and width in the batch and optionally returns - their corresponding pixel mask. - - Args: - images (`List[np.ndarray]`): - Batch of images to pad. - return_tensors (`str` or `TensorType`, *optional*): - The type of tensors to return. Can be one of: - - Unset: Return a list of `np.ndarray`. - - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`. - - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. - - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. - - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format of the image. If not provided, it will be the same as the input image. - """ - pad_size = get_max_dimensions(images) - padded_images = [pad(image=image, output_size=pad_size, data_format=data_format) for image in images] - data = {"pixel_values": padded_images} - - return BatchFeature(data=data, tensor_type=return_tensors) - - def pad_and_create_pixel_mask( - self, - pixel_values_list: List[ImageInput], - return_tensors: Optional[Union[str, TensorType]] = None, - data_format: Optional[ChannelDimension] = None, - ) -> BatchFeature: - """ - Pads a batch of images with zeros to the size of largest height and width in the batch and returns their - corresponding pixel mask. - - Args: - images (`List[np.ndarray]`): - Batch of images to pad. - return_tensors (`str` or `TensorType`, *optional*): - The type of tensors to return. Can be one of: - - Unset: Return a list of `np.ndarray`. - - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`. - - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. - - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. - - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format of the image. If not provided, it will be the same as the input image. - """ - warnings.warn( - "This method is deprecated and will be removed in v4.26.0. Please use pad instead.", FutureWarning - ) - # pad expects a list of np.ndarray, but the previous feature extractors expected torch tensors - images = [to_numpy_array(image) for image in pixel_values_list] - return self.pad( - images=images, - return_tensors=return_tensors, - data_format=data_format, - ) def preprocess( self, @@ -408,7 +300,6 @@ def preprocess( do_normalize: Optional[bool] = None, image_mean: Optional[Union[float, List[float]]] = None, image_std: Optional[Union[float, List[float]]] = None, - do_pad: Optional[bool] = None, return_tensors: Optional[Union[str, TensorType]] = None, do_convert_rgb: bool = None, data_format: ChannelDimension = ChannelDimension.FIRST, @@ -445,9 +336,6 @@ def preprocess( Image mean to normalize the image by if `do_normalize` is set to `True`. image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`): Image standard deviation to normalize the image by if `do_normalize` is set to `True`. - do_pad (`bool`, *optional*, defaults to `self.do_pad`): - Whether to pad the image to the (max_height, max_width) in the batch. If `True`, a pixel mask is also - created and returned. do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`): Whether to convert the image to RGB. return_tensors (`str` or `TensorType`, *optional*): @@ -473,7 +361,6 @@ def preprocess( do_normalize = do_normalize if do_normalize is not None else self.do_normalize image_mean = image_mean if image_mean is not None else self.image_mean image_std = image_std if image_std is not None else self.image_std - do_pad = do_pad if do_pad is not None else self.do_pad do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb size = size if size is not None else self.size @@ -518,9 +405,6 @@ def preprocess( images = [to_channel_dimension_format(image, data_format) for image in images] - if do_pad: - encoded_outputs = self.pad(images, return_tensors=return_tensors) - else: - encoded_outputs = BatchFeature(data={"pixel_values": images}, tensor_type=return_tensors) + encoded_outputs = BatchFeature(data={"pixel_values": images}, tensor_type=return_tensors) return encoded_outputs From 6e9e86f920d6f576a9f66a4feb021dcb15656143 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Wed, 14 Dec 2022 09:11:56 +0000 Subject: [PATCH 79/96] remove `crop` and `center_crop` --- .../models/blip/image_processing_blip.py | 45 +------------------ .../models/blip/test_image_processing_blip.py | 43 +++++++----------- 2 files changed, 18 insertions(+), 70 deletions(-) diff --git a/src/transformers/models/blip/image_processing_blip.py b/src/transformers/models/blip/image_processing_blip.py index bae5ee61dd3d..970437a839e5 100644 --- a/src/transformers/models/blip/image_processing_blip.py +++ b/src/transformers/models/blip/image_processing_blip.py @@ -23,7 +23,7 @@ from transformers.utils.generic import TensorType from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict -from ...image_transforms import center_crop, normalize, rescale, resize, to_channel_dimension_format +from ...image_transforms import normalize, rescale, resize, to_channel_dimension_format from ...image_utils import ( IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD, @@ -159,8 +159,6 @@ def __init__( size: Dict[str, int] = None, size_divisor: int = 32, resample: PILImageResampling = PILImageResampling.BICUBIC, - do_center_crop: bool = False, - crop_size: Dict[str, int] = None, do_rescale: bool = True, rescale_factor: Union[int, float] = 1 / 255, do_normalize: bool = True, @@ -171,16 +169,12 @@ def __init__( ) -> None: super().__init__(**kwargs) - size = size if size is not None else 384 + size = size if size is not None else {"height": 384, "width": 384} size = get_size_dict(size, default_to_square=True) - crop_size = crop_size if crop_size is not None else {"height": 224, "width": 224} - crop_size = get_size_dict(crop_size, default_to_square=True, param_name="crop_size") self.do_resize = do_resize self.size = size self.size_divisor = size_divisor - self.do_center_crop = do_center_crop - self.crop_size = crop_size self.resample = resample self.do_rescale = do_rescale self.rescale_factor = rescale_factor @@ -189,29 +183,6 @@ def __init__( self.image_std = image_std if image_std is not None else IMAGENET_STANDARD_STD self.do_convert_rgb = do_convert_rgb - def center_crop( - self, - image: np.ndarray, - size: Dict[str, int], - data_format: Optional[Union[str, ChannelDimension]] = None, - **kwargs - ) -> np.ndarray: - """ - Center crop an image. If the image is too small to be cropped to the size given, it will be padded (so the - returned result will always be of size `size`). - - Args: - image (`np.ndarray`): - Image to center crop. - size (`Dict[str, int]`): - Size of the output image in the form of a dictionary with keys `height` and `width`. - data_format (`str` or `ChannelDimension`, *optional*): - The channel dimension format of the image. If not provided, it will be the same as the input image. - """ - size = get_size_dict(size) - if "height" not in size or "width" not in size: - raise ValueError(f"The `size` parameter must contain the keys (height, width). Got {size.keys()}") - return center_crop(image, size=(size["height"], size["width"]), data_format=data_format, **kwargs) def resize( self, @@ -294,8 +265,6 @@ def preprocess( size_divisor: Optional[int] = None, resample: PILImageResampling = None, do_rescale: Optional[bool] = None, - do_center_crop: bool = None, - crop_size: int = None, rescale_factor: Optional[float] = None, do_normalize: Optional[bool] = None, image_mean: Optional[Union[float, List[float]]] = None, @@ -322,10 +291,6 @@ def preprocess( The image is resized to a size that is a multiple of this value. resample (`PILImageResampling`, *optional*, defaults to `self.resample`): Resampling filter to use if resizing the image. Only has an effect if `do_resize` is set to `True`. - do_center_crop (`bool`, *optional*, defaults to `self.do_center_crop`): - Whether to center crop the image. - crop_size (`Dict[str, int]`, *optional*, defaults to `self.crop_size`): - Size of the center crop. Only has an effect if `do_center_crop` is set to `True`. do_rescale (`bool`, *optional*, defaults to `self.do_rescale`): Whether to rescale the image values between [0 - 1]. rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`): @@ -353,9 +318,6 @@ def preprocess( do_resize = do_resize if do_resize is not None else self.do_resize size_divisor = size_divisor if size_divisor is not None else self.size_divisor resample = resample if resample is not None else self.resample - do_center_crop = do_center_crop if do_center_crop is not None else self.do_center_crop - crop_size = crop_size if crop_size is not None else self.crop_size - crop_size = get_size_dict(crop_size, param_name="crop_size", default_to_square=True) do_rescale = do_rescale if do_rescale is not None else self.do_rescale rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor do_normalize = do_normalize if do_normalize is not None else self.do_normalize @@ -394,9 +356,6 @@ def preprocess( if do_resize: images = [self.resize(image=image, size=size, resample=resample) for image in images] - if do_center_crop: - images = [self.center_crop(image=image, size=crop_size) for image in images] - if do_rescale: images = [self.rescale(image=image, scale=rescale_factor) for image in images] diff --git a/tests/models/blip/test_image_processing_blip.py b/tests/models/blip/test_image_processing_blip.py index f8edad9a237a..ea31038b14ab 100644 --- a/tests/models/blip/test_image_processing_blip.py +++ b/tests/models/blip/test_image_processing_blip.py @@ -44,8 +44,6 @@ def __init__( max_resolution=400, do_resize=True, size=None, - do_center_crop=True, - crop_size=None, do_normalize=True, do_pad=False, image_mean=[0.48145466, 0.4578275, 0.40821073], @@ -53,7 +51,6 @@ def __init__( do_convert_rgb=True, ): size = size if size is not None else {"height": 20, "width": 20} - crop_size = crop_size if crop_size is not None else {"height": 18, "width": 18} self.parent = parent self.batch_size = batch_size self.num_channels = num_channels @@ -62,8 +59,6 @@ def __init__( self.max_resolution = max_resolution self.do_resize = do_resize self.size = size - self.do_center_crop = do_center_crop - self.crop_size = crop_size self.do_normalize = do_normalize self.image_mean = image_mean self.image_std = image_std @@ -74,8 +69,6 @@ def prepare_feat_extract_dict(self): return { "do_resize": self.do_resize, "size": self.size, - "do_center_crop": self.do_center_crop, - "crop_size": self.crop_size, "do_normalize": self.do_normalize, "image_mean": self.image_mean, "image_std": self.image_std, @@ -131,8 +124,6 @@ def test_feat_extract_properties(self): feature_extractor = self.feature_extraction_class(**self.feat_extract_dict) self.assertTrue(hasattr(feature_extractor, "do_resize")) self.assertTrue(hasattr(feature_extractor, "size")) - self.assertTrue(hasattr(feature_extractor, "do_center_crop")) - self.assertTrue(hasattr(feature_extractor, "center_crop")) self.assertTrue(hasattr(feature_extractor, "do_normalize")) self.assertTrue(hasattr(feature_extractor, "image_mean")) self.assertTrue(hasattr(feature_extractor, "image_std")) @@ -156,8 +147,8 @@ def test_call_pil(self): ( 1, self.feature_extract_tester.num_channels, - self.feature_extract_tester.crop_size["height"], - self.feature_extract_tester.crop_size["width"], + self.feature_extract_tester.size["height"], + self.feature_extract_tester.size["width"], ), ) @@ -168,8 +159,8 @@ def test_call_pil(self): ( self.feature_extract_tester.batch_size, self.feature_extract_tester.num_channels, - self.feature_extract_tester.crop_size["height"], - self.feature_extract_tester.crop_size["width"], + self.feature_extract_tester.size["height"], + self.feature_extract_tester.size["width"], ), ) @@ -188,8 +179,8 @@ def test_call_numpy(self): ( 1, self.feature_extract_tester.num_channels, - self.feature_extract_tester.crop_size["height"], - self.feature_extract_tester.crop_size["width"], + self.feature_extract_tester.size["height"], + self.feature_extract_tester.size["width"], ), ) @@ -200,8 +191,8 @@ def test_call_numpy(self): ( self.feature_extract_tester.batch_size, self.feature_extract_tester.num_channels, - self.feature_extract_tester.crop_size["height"], - self.feature_extract_tester.crop_size["width"], + self.feature_extract_tester.size["height"], + self.feature_extract_tester.size["width"], ), ) @@ -220,8 +211,8 @@ def test_call_pytorch(self): ( 1, self.feature_extract_tester.num_channels, - self.feature_extract_tester.crop_size["height"], - self.feature_extract_tester.crop_size["width"], + self.feature_extract_tester.size["height"], + self.feature_extract_tester.size["width"], ), ) @@ -232,8 +223,8 @@ def test_call_pytorch(self): ( self.feature_extract_tester.batch_size, self.feature_extract_tester.num_channels, - self.feature_extract_tester.crop_size["height"], - self.feature_extract_tester.crop_size["width"], + self.feature_extract_tester.size["height"], + self.feature_extract_tester.size["width"], ), ) @@ -256,8 +247,6 @@ def test_feat_extract_properties(self): feature_extractor = self.feature_extraction_class(**self.feat_extract_dict) self.assertTrue(hasattr(feature_extractor, "do_resize")) self.assertTrue(hasattr(feature_extractor, "size")) - self.assertTrue(hasattr(feature_extractor, "do_center_crop")) - self.assertTrue(hasattr(feature_extractor, "center_crop")) self.assertTrue(hasattr(feature_extractor, "do_normalize")) self.assertTrue(hasattr(feature_extractor, "image_mean")) self.assertTrue(hasattr(feature_extractor, "image_std")) @@ -281,8 +270,8 @@ def test_call_pil_four_channels(self): ( 1, self.expected_encoded_image_num_channels, - self.feature_extract_tester.crop_size["height"], - self.feature_extract_tester.crop_size["width"], + self.feature_extract_tester.size["height"], + self.feature_extract_tester.size["width"], ), ) @@ -293,7 +282,7 @@ def test_call_pil_four_channels(self): ( self.feature_extract_tester.batch_size, self.expected_encoded_image_num_channels, - self.feature_extract_tester.crop_size["height"], - self.feature_extract_tester.crop_size["width"], + self.feature_extract_tester.size["height"], + self.feature_extract_tester.size["width"], ), ) From d5115fa6fda841caa18ebe6028dca202004a694b Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Wed, 14 Dec 2022 10:12:46 +0100 Subject: [PATCH 80/96] Update src/transformers/models/blip/image_processing_blip.py Co-authored-by: NielsRogge <48327001+NielsRogge@users.noreply.github.com> --- src/transformers/models/blip/image_processing_blip.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/blip/image_processing_blip.py b/src/transformers/models/blip/image_processing_blip.py index 970437a839e5..39971ba89d88 100644 --- a/src/transformers/models/blip/image_processing_blip.py +++ b/src/transformers/models/blip/image_processing_blip.py @@ -121,8 +121,9 @@ class BlipImageProcessor(BaseImageProcessor): do_resize (`bool`, *optional*, defaults to `True`): Whether to resize the image's (height, width) dimensions to the specified `size`. Can be overridden by the `do_resize` parameter in the `preprocess` method. - size (`Dict[str, int]`, *optional*, defaults to `384`): - Resize the input to the given size. Only has an effect if `do_resize` is set to `True`. + size (`dict`, *optional*, defaults to `{"height": 384, "width": 384}`): + Size of the output image after resizing. Can be overridden by the `size` parameter in the `preprocess` + method. size_divisor (`int`, *optional*, defaults to 32): The size by which to make sure both the height and width can be divided. Only has an effect if `do_resize` is set to `True`. Can be overridden by the `size_divisor` parameter in the `preprocess` method. From 5fd122aefda0c91de2868fe4bc51fbbc359d1744 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Wed, 14 Dec 2022 09:13:45 +0000 Subject: [PATCH 81/96] fix --- .../models/blip/image_processing_blip.py | 33 ------------------- 1 file changed, 33 deletions(-) diff --git a/src/transformers/models/blip/image_processing_blip.py b/src/transformers/models/blip/image_processing_blip.py index 970437a839e5..6cf4b5f87dd9 100644 --- a/src/transformers/models/blip/image_processing_blip.py +++ b/src/transformers/models/blip/image_processing_blip.py @@ -85,33 +85,6 @@ def get_max_dimensions(images: List[np.ndarray]) -> List[int]: return (max_height, max_width) -# Copied from transformers.models.vilt.image_processing_vilt.get_resize_output_image_size -def get_resize_output_image_size( - input_image: np.ndarray, shorter: int = 800, longer: int = 1333, size_divisor: int = 32 -) -> Tuple[int, int]: - input_height, input_width = get_image_size(input_image) - min_size, max_size = shorter, longer - - scale = min_size / min(input_height, input_width) - - if input_height < input_width: - new_height = min_size - new_width = scale * input_width - else: - new_height = scale * input_height - new_width = min_size - - if max(new_height, new_width) > max_size: - scale = max_size / max(new_height, new_width) - new_height = scale * new_height - new_width = scale * new_width - - new_height, new_width = int(new_height + 0.5), int(new_width + 0.5) - new_height = new_height // size_divisor * size_divisor - new_width = new_width // size_divisor * size_divisor - - return new_height, new_width - class BlipImageProcessor(BaseImageProcessor): r""" @@ -123,9 +96,6 @@ class BlipImageProcessor(BaseImageProcessor): `do_resize` parameter in the `preprocess` method. size (`Dict[str, int]`, *optional*, defaults to `384`): Resize the input to the given size. Only has an effect if `do_resize` is set to `True`. - size_divisor (`int`, *optional*, defaults to 32): - The size by which to make sure both the height and width can be divided. Only has an effect if `do_resize` - is set to `True`. Can be overridden by the `size_divisor` parameter in the `preprocess` method. resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`): Resampling filter to use if resizing the image. Only has an effect if `do_resize` is set to `True`. Can be overridden by the `resample` parameter in the `preprocess` method. @@ -157,7 +127,6 @@ def __init__( self, do_resize: bool = True, size: Dict[str, int] = None, - size_divisor: int = 32, resample: PILImageResampling = PILImageResampling.BICUBIC, do_rescale: bool = True, rescale_factor: Union[int, float] = 1 / 255, @@ -174,7 +143,6 @@ def __init__( self.do_resize = do_resize self.size = size - self.size_divisor = size_divisor self.resample = resample self.do_rescale = do_rescale self.rescale_factor = rescale_factor @@ -262,7 +230,6 @@ def preprocess( images: ImageInput, do_resize: Optional[bool] = None, size: Optional[Dict[str, int]] = None, - size_divisor: Optional[int] = None, resample: PILImageResampling = None, do_rescale: Optional[bool] = None, rescale_factor: Optional[float] = None, From d4aab8927994d006611876d1ecefee0aa0cf08df Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Wed, 14 Dec 2022 09:16:13 +0000 Subject: [PATCH 82/96] remove `size_divisor` --- .../models/blip/image_processing_blip.py | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/src/transformers/models/blip/image_processing_blip.py b/src/transformers/models/blip/image_processing_blip.py index c68ccb7fe7b9..aaaa08505297 100644 --- a/src/transformers/models/blip/image_processing_blip.py +++ b/src/transformers/models/blip/image_processing_blip.py @@ -14,8 +14,7 @@ # limitations under the License. """Image processor class for BLIP.""" -import warnings -from typing import Any, Dict, Iterable, List, Optional, Tuple, Union +from typing import Any, Dict, Iterable, List, Optional, Union import numpy as np @@ -30,7 +29,6 @@ ChannelDimension, ImageInput, PILImageResampling, - get_image_size, infer_channel_dimension_format, is_batched, to_numpy_array, @@ -85,7 +83,6 @@ def get_max_dimensions(images: List[np.ndarray]) -> List[int]: return (max_height, max_width) - class BlipImageProcessor(BaseImageProcessor): r""" Constructs a BLIP image processor. @@ -94,17 +91,9 @@ class BlipImageProcessor(BaseImageProcessor): do_resize (`bool`, *optional*, defaults to `True`): Whether to resize the image's (height, width) dimensions to the specified `size`. Can be overridden by the `do_resize` parameter in the `preprocess` method. -<<<<<<< HEAD - size (`Dict[str, int]`, *optional*, defaults to `384`): - Resize the input to the given size. Only has an effect if `do_resize` is set to `True`. -======= size (`dict`, *optional*, defaults to `{"height": 384, "width": 384}`): Size of the output image after resizing. Can be overridden by the `size` parameter in the `preprocess` method. - size_divisor (`int`, *optional*, defaults to 32): - The size by which to make sure both the height and width can be divided. Only has an effect if `do_resize` - is set to `True`. Can be overridden by the `size_divisor` parameter in the `preprocess` method. ->>>>>>> d5115fa6fda841caa18ebe6028dca202004a694b resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`): Resampling filter to use if resizing the image. Only has an effect if `do_resize` is set to `True`. Can be overridden by the `resample` parameter in the `preprocess` method. @@ -160,7 +149,6 @@ def __init__( self.image_std = image_std if image_std is not None else IMAGENET_STANDARD_STD self.do_convert_rgb = do_convert_rgb - def resize( self, image: np.ndarray, @@ -233,7 +221,6 @@ def normalize( """ return normalize(image, mean=mean, std=std, data_format=data_format, **kwargs) - def preprocess( self, images: ImageInput, @@ -263,8 +250,6 @@ def preprocess( `size["shortest_edge"]` whilst preserving the aspect ratio. If the longest edge of this resized image is > `int(size["shortest_edge"] * (1333 / 800))`, then the image is resized again to make the longest edge equal to `int(size["shortest_edge"] * (1333 / 800))`. - size_divisor (`int`, *optional*, defaults to `self.size_divisor`): - The image is resized to a size that is a multiple of this value. resample (`PILImageResampling`, *optional*, defaults to `self.resample`): Resampling filter to use if resizing the image. Only has an effect if `do_resize` is set to `True`. do_rescale (`bool`, *optional*, defaults to `self.do_rescale`): @@ -292,7 +277,6 @@ def preprocess( - `ChannelDimension.LAST`: image in (height, width, num_channels) format. """ do_resize = do_resize if do_resize is not None else self.do_resize - size_divisor = size_divisor if size_divisor is not None else self.size_divisor resample = resample if resample is not None else self.resample do_rescale = do_rescale if do_rescale is not None else self.do_rescale rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor From 00fd12dc65ea3992cd53375f968bf4b96169a207 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Wed, 14 Dec 2022 15:37:46 +0000 Subject: [PATCH 83/96] fix weights `init` --- .../models/blip/configuration_blip.py | 3 +- src/transformers/models/blip/modeling_blip.py | 63 +++++++------------ tests/models/blip/test_modeling_blip.py | 2 +- 3 files changed, 25 insertions(+), 43 deletions(-) diff --git a/src/transformers/models/blip/configuration_blip.py b/src/transformers/models/blip/configuration_blip.py index b450f839e4f4..7c337cb6645d 100644 --- a/src/transformers/models/blip/configuration_blip.py +++ b/src/transformers/models/blip/configuration_blip.py @@ -252,7 +252,7 @@ def __init__( layer_norm_eps=0.00001, dropout=0.0, attention_dropout=0.0, - initializer_range=0.02, + initializer_range=1e-10, initializer_factor=1.0, **kwargs ): @@ -374,6 +374,7 @@ def __init__( self.projection_dim = projection_dim self.logit_scale_init_value = logit_scale_init_value self.initializer_factor = 1.0 + self.initializer_range = 0.02 self.image_text_hidden_size = image_text_hidden_size @classmethod diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 4c77b62d4992..b440eeb54812 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -222,7 +222,13 @@ def __init__(self, config: BlipVisionConfig): self.image_size = config.image_size self.patch_size = config.patch_size - self.class_embedding = nn.Parameter(torch.randn(1, 1, self.embed_dim)) + self.class_embedding = nn.Parameter( + nn.init.trunc_normal_( + torch.zeros(1, 1, self.embed_dim, dtype=torch.float32), + mean=0.0, + std=config.initializer_range, + ) + ) self.patch_embedding = nn.Conv2d( in_channels=3, out_channels=self.embed_dim, kernel_size=self.patch_size, stride=self.patch_size @@ -230,16 +236,24 @@ def __init__(self, config: BlipVisionConfig): self.num_patches = (self.image_size // self.patch_size) ** 2 self.num_positions = self.num_patches + 1 - self.position_embedding = nn.Parameter(torch.zeros(1, self.num_positions, self.embed_dim)) + + self.position_embedding = nn.Parameter( + nn.init.trunc_normal_( + torch.zeros(1, self.num_positions, self.embed_dim, dtype=torch.float32), + mean=0.0, + std=config.initializer_range, + ) + ) def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: batch_size = pixel_values.shape[0] + target_dtype = self.patch_embedding.weight.dtype patch_embeds = self.patch_embedding(pixel_values) # shape = [*, width, grid, grid] patch_embeds = patch_embeds.flatten(2).transpose(1, 2) - class_embeds = self.class_embedding.expand(batch_size, 1, -1) + class_embeds = self.class_embedding.expand(batch_size, 1, -1).to(target_dtype) embeddings = torch.cat([class_embeds, patch_embeds], dim=1) - embeddings = embeddings + self.position_embedding[:, : embeddings.size(1), :] + embeddings = embeddings + self.position_embedding[:, : embeddings.size(1), :].to(target_dtype) return embeddings @@ -426,44 +440,11 @@ class BlipPreTrainedModel(PreTrainedModel): def _init_weights(self, module): """Initialize the weights""" - factor = self.config.initializer_factor - if isinstance(module, BlipTextEmbeddings): - module.token_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02) - module.position_embedding.weight.data.normal_(mean=0.0, std=factor * 0.02) - elif isinstance(module, BlipVisionEmbeddings): - module.class_embedding.data.normal_(mean=0.0, std=factor * 0.02) - module.patch_embedding.weight.data.normal_(mean=0.0, std=module.config.initializer_range * factor) - module.patch_embedding.bias.data.normal_(mean=0.0, std=module.config.initializer_range * factor) - elif isinstance(module, BlipAttention): - factor = self.config.initializer_factor - in_proj_std = (module.embed_dim**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor - nn.init.normal_(module.qkv.weight, std=in_proj_std) - if module.qkv.bias is not None: - nn.init.normal_(module.qkv.bias, std=in_proj_std) - nn.init.normal_(module.projection.weight, std=in_proj_std) - if module.projection.bias is not None: - nn.init.normal_(module.projection.bias, std=in_proj_std) - elif isinstance(module, BlipMLP): - factor = self.config.initializer_factor - in_proj_std = ( - (module.config.hidden_size**-0.5) * ((2 * module.config.num_hidden_layers) ** -0.5) * factor - ) - fc_std = (2 * module.config.hidden_size) ** -0.5 * factor - nn.init.normal_(module.fc1.weight, std=fc_std) - nn.init.normal_(module.fc2.weight, std=in_proj_std) - elif isinstance(module, BlipModel): - nn.init.normal_( - module.text_projection.weight, - std=module.text_embed_dim**-0.5 * self.config.initializer_factor, - ) - nn.init.normal_( - module.visual_projection.weight, - std=module.vision_embed_dim**-0.5 * self.config.initializer_factor, - ) - elif isinstance(module, nn.Conv2d) or isinstance(module, nn.Embedding) or isinstance(module, nn.Linear): - nn.init.normal_(module.weight, mean=0.0, std=factor * 0.02) + factor = self.config.initializer_range + if isinstance(module, nn.Conv2d) or isinstance(module, nn.Embedding) or isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=factor) if hasattr(module, "bias") and module.bias is not None: - nn.init.normal_(module.bias, mean=0.0, std=factor * 0.02) + module.bias.data.zero_() elif isinstance(module, nn.LayerNorm): module.bias.data.zero_() diff --git a/tests/models/blip/test_modeling_blip.py b/tests/models/blip/test_modeling_blip.py index af17cbcc3c92..9db08bad03c5 100644 --- a/tests/models/blip/test_modeling_blip.py +++ b/tests/models/blip/test_modeling_blip.py @@ -74,7 +74,7 @@ def __init__( intermediate_size=37, dropout=0.1, attention_dropout=0.1, - initializer_range=0.02, + initializer_range=1e-10, scope=None, ): self.parent = parent From 4577db8cf5ac549ae07f0d9609dd7bff505ad62f Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Thu, 15 Dec 2022 14:10:10 +0000 Subject: [PATCH 84/96] remove unneeded functions --- .../models/blip/image_processing_blip.py | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/src/transformers/models/blip/image_processing_blip.py b/src/transformers/models/blip/image_processing_blip.py index aaaa08505297..e617133ccc8b 100644 --- a/src/transformers/models/blip/image_processing_blip.py +++ b/src/transformers/models/blip/image_processing_blip.py @@ -59,30 +59,6 @@ def convert_to_rgb(image: Union[Any, PIL.Image.Image]) -> Union[Any, PIL.Image.I return image.convert("RGB") -# Copied from transformers.models.vilt.image_processing_vilt.max_across_indices -def max_across_indices(values: Iterable[Any]) -> List[Any]: - """ - Return the maximum value across all indices of an iterable of values. - """ - return [max(values_i) for values_i in zip(*values)] - - -# Copied from transformers.models.vilt.image_processing_vilt.get_max_dimensions -def get_max_dimensions(images: List[np.ndarray]) -> List[int]: - """ - Get the maximum height and width across all images in a batch. - """ - input_channel_dimension = infer_channel_dimension_format(images[0]) - - if input_channel_dimension == ChannelDimension.FIRST: - _, max_height, max_width = max_across_indices([img.shape for img in images]) - elif input_channel_dimension == ChannelDimension.LAST: - max_height, max_width, _ = max_across_indices([img.shape for img in images]) - else: - raise ValueError(f"Invalid channel dimension format: {input_channel_dimension}") - return (max_height, max_width) - - class BlipImageProcessor(BaseImageProcessor): r""" Constructs a BLIP image processor. From 8cdc0d9c231805c860ce4f4f662822ae60c4a398 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Thu, 15 Dec 2022 14:20:25 +0000 Subject: [PATCH 85/96] add suggestions --- docs/source/en/model_doc/blip.mdx | 23 +++++++++---------- .../models/blip/image_processing_blip.py | 6 ++--- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/docs/source/en/model_doc/blip.mdx b/docs/source/en/model_doc/blip.mdx index de42029b4476..81f51bfd688a 100644 --- a/docs/source/en/model_doc/blip.mdx +++ b/docs/source/en/model_doc/blip.mdx @@ -45,6 +45,16 @@ The original code can be found [here](https://github.com/salesforce/BLIP). [[autodoc]] BlipVisionConfig +## BlipProcessor + +[[autodoc]] BlipProcessor + + +## BlipImageProcessor + +[[autodoc]] BlipImageProcessor + - preprocess + ## BlipModel [[autodoc]] BlipModel @@ -79,15 +89,4 @@ The original code can be found [here](https://github.com/salesforce/BLIP). ## BlipForQuestionAnswering [[autodoc]] BlipForQuestionAnswering - - forward - - -## BlipProcessor - -[[autodoc]] BlipProcessor - - -## BlipImageProcessor - -[[autodoc]] BlipImageProcessor - - preprocess \ No newline at end of file + - forward \ No newline at end of file diff --git a/src/transformers/models/blip/image_processing_blip.py b/src/transformers/models/blip/image_processing_blip.py index e617133ccc8b..a1152bc7f10e 100644 --- a/src/transformers/models/blip/image_processing_blip.py +++ b/src/transformers/models/blip/image_processing_blip.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for BLIP.""" -from typing import Any, Dict, Iterable, List, Optional, Union +from typing import Any, Dict, List, Optional, Union import numpy as np @@ -29,7 +29,6 @@ ChannelDimension, ImageInput, PILImageResampling, - infer_channel_dimension_format, is_batched, to_numpy_array, valid_images, @@ -91,8 +90,7 @@ class BlipImageProcessor(BaseImageProcessor): number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method. Can be overridden by the `image_std` parameter in the `preprocess` method. do_convert_rgb (`bool`, *optional*, defaults to `True`): - Standard deviation to use if normalizing the image. This is a float or list of floats the length of the - number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method. + Whether to convert the image to RGB. """ model_input_names = ["pixel_values"] From b106ef1cdcfee89acda676197e4bad18130702a3 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Thu, 15 Dec 2022 14:34:29 +0000 Subject: [PATCH 86/96] minor changes - change slow test output for PT 1.13 - docstring order --- src/transformers/models/blip/modeling_blip.py | 4 ++-- tests/models/blip/test_modeling_blip.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index b440eeb54812..58ca23401142 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -74,10 +74,10 @@ class BlipForConditionalGenerationModelOutput(ModelOutput): Args: loss (`torch.FloatTensor`, *optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`): Languge modeling loss from the text decoder. - image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)`, *optional*): - The image embeddings obtained after applying the Vision Transformer model to the input image. decoder_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`, *optional*): Prediction scores of the language modeling head of the text decoder model. + image_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)`, *optional*): + The image embeddings obtained after applying the Vision Transformer model to the input image. last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the model. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True`): diff --git a/tests/models/blip/test_modeling_blip.py b/tests/models/blip/test_modeling_blip.py index 9db08bad03c5..57c0cae872bf 100644 --- a/tests/models/blip/test_modeling_blip.py +++ b/tests/models/blip/test_modeling_blip.py @@ -826,5 +826,7 @@ def test_inference_itm(self): out_itm = model(**inputs) out = model(**inputs, use_itm_head=False) - self.assertTrue(torch.allclose(out_itm[0].cpu(), torch.Tensor([[1.9452, -1.9378]]), atol=1e-3, rtol=1e-3)) + expected_scores = torch.Tensor([[0.9779, 0.0221]]) + + self.assertTrue(torch.allclose(torch.nn.Softmax()(out_itm[0].cpu()), expected_scores, atol=1e-3, rtol=1e-3)) self.assertTrue(torch.allclose(out[0].cpu(), torch.Tensor([[0.5053]]), atol=1e-3, rtol=1e-3)) From 7ba9cb0f1b5455b5edfd5f0516ee1b84a106c4f2 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Thu, 15 Dec 2022 14:39:28 +0000 Subject: [PATCH 87/96] replace `feature_extractor` by `image_processor` --- .../models/blip/processing_blip.py | 23 ++++++------ tests/models/blip/test_processor_blip.py | 36 +++++++++---------- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/src/transformers/models/blip/processing_blip.py b/src/transformers/models/blip/processing_blip.py index a65e90bb4d86..9916299179f4 100644 --- a/src/transformers/models/blip/processing_blip.py +++ b/src/transformers/models/blip/processing_blip.py @@ -31,18 +31,19 @@ class BlipProcessor(ProcessorMixin): docstring of [`~BlipProcessor.__call__`] and [`~BlipProcessor.decode`] for more information. Args: - feature_extractor (`BlipImageProcessor`): - An instance of [`BlipImageProcessor`]. The feature extractor is a required input. + image_processor (`BlipImageProcessor`): + An instance of [`BlipImageProcessor`]. The image processor is a required input. tokenizer (`BertTokenizerFast`): An instance of ['BertTokenizerFast`]. The tokenizer is a required input. """ - feature_extractor_class = "BlipImageProcessor" + attributes = ["image_processor", "tokenizer"] + image_processor_class = "BlipImageProcessor" tokenizer_class = ("BertTokenizer", "BertTokenizerFast") - def __init__(self, feature_extractor, tokenizer): + def __init__(self, image_processor, tokenizer): tokenizer.return_token_type_ids = False - super().__init__(feature_extractor, tokenizer) - self.current_processor = self.feature_extractor + super().__init__(image_processor, tokenizer) + self.current_processor = self.image_processor def __call__( self, @@ -99,7 +100,7 @@ def __call__( return text_encoding # add pixel_values + pixel_mask - encoding_feature_extractor = self.feature_extractor(images, return_tensors=return_tensors) + encoding_image_processor = self.image_processor(images, return_tensors=return_tensors) if text is not None: text_encoding = self.tokenizer( @@ -124,9 +125,9 @@ def __call__( text_encoding = None if text_encoding is not None: - encoding_feature_extractor.update(text_encoding) + encoding_image_processor.update(text_encoding) - return encoding_feature_extractor + return encoding_image_processor def batch_decode(self, *args, **kwargs): """ @@ -145,5 +146,5 @@ def decode(self, *args, **kwargs): @property def model_input_names(self): tokenizer_input_names = self.tokenizer.model_input_names - feature_extractor_input_names = self.feature_extractor.model_input_names - return list(dict.fromkeys(tokenizer_input_names + feature_extractor_input_names)) + image_processor_input_names = self.image_processor.model_input_names + return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names)) diff --git a/tests/models/blip/test_processor_blip.py b/tests/models/blip/test_processor_blip.py index 15f4ea5251dc..b6d8b2e70175 100644 --- a/tests/models/blip/test_processor_blip.py +++ b/tests/models/blip/test_processor_blip.py @@ -43,8 +43,8 @@ def setUp(self): def get_tokenizer(self, **kwargs): return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).tokenizer - def get_feature_extractor(self, **kwargs): - return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).feature_extractor + def get_image_processor(self, **kwargs): + return AutoProcessor.from_pretrained(self.tmpdirname, **kwargs).image_processor def tearDown(self): shutil.rmtree(self.tmpdirname) @@ -61,11 +61,11 @@ def prepare_image_inputs(self): return image_inputs def test_save_load_pretrained_additional_features(self): - processor = BlipProcessor(tokenizer=self.get_tokenizer(), feature_extractor=self.get_feature_extractor()) + processor = BlipProcessor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor()) processor.save_pretrained(self.tmpdirname) tokenizer_add_kwargs = self.get_tokenizer(bos_token="(BOS)", eos_token="(EOS)") - feature_extractor_add_kwargs = self.get_feature_extractor(do_normalize=False, padding_value=1.0) + image_processor_add_kwargs = self.get_image_processor(do_normalize=False, padding_value=1.0) processor = BlipProcessor.from_pretrained( self.tmpdirname, bos_token="(BOS)", eos_token="(EOS)", do_normalize=False, padding_value=1.0 @@ -74,28 +74,28 @@ def test_save_load_pretrained_additional_features(self): self.assertEqual(processor.tokenizer.get_vocab(), tokenizer_add_kwargs.get_vocab()) self.assertIsInstance(processor.tokenizer, PreTrainedTokenizerFast) - self.assertEqual(processor.feature_extractor.to_json_string(), feature_extractor_add_kwargs.to_json_string()) - self.assertIsInstance(processor.feature_extractor, BlipImageProcessor) + self.assertEqual(processor.image_processor.to_json_string(), image_processor_add_kwargs.to_json_string()) + self.assertIsInstance(processor.image_processor, BlipImageProcessor) - def test_feature_extractor(self): - feature_extractor = self.get_feature_extractor() + def test_image_processor(self): + image_processor = self.get_image_processor() tokenizer = self.get_tokenizer() - processor = BlipProcessor(tokenizer=tokenizer, feature_extractor=feature_extractor) + processor = BlipProcessor(tokenizer=tokenizer, image_processor=image_processor) image_input = self.prepare_image_inputs() - input_feat_extract = feature_extractor(image_input, return_tensors="np") + input_feat_extract = image_processor(image_input, return_tensors="np") input_processor = processor(images=image_input, return_tensors="np") for key in input_feat_extract.keys(): self.assertAlmostEqual(input_feat_extract[key].sum(), input_processor[key].sum(), delta=1e-2) def test_tokenizer(self): - feature_extractor = self.get_feature_extractor() + image_processor = self.get_image_processor() tokenizer = self.get_tokenizer() - processor = BlipProcessor(tokenizer=tokenizer, feature_extractor=feature_extractor) + processor = BlipProcessor(tokenizer=tokenizer, image_processor=image_processor) input_str = "lower newer" @@ -107,10 +107,10 @@ def test_tokenizer(self): self.assertListEqual(encoded_tok[key], encoded_processor[key]) def test_processor(self): - feature_extractor = self.get_feature_extractor() + image_processor = self.get_image_processor() tokenizer = self.get_tokenizer() - processor = BlipProcessor(tokenizer=tokenizer, feature_extractor=feature_extractor) + processor = BlipProcessor(tokenizer=tokenizer, image_processor=image_processor) input_str = "lower newer" image_input = self.prepare_image_inputs() @@ -124,10 +124,10 @@ def test_processor(self): processor() def test_tokenizer_decode(self): - feature_extractor = self.get_feature_extractor() + image_processor = self.get_image_processor() tokenizer = self.get_tokenizer() - processor = BlipProcessor(tokenizer=tokenizer, feature_extractor=feature_extractor) + processor = BlipProcessor(tokenizer=tokenizer, image_processor=image_processor) predicted_ids = [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]] @@ -137,10 +137,10 @@ def test_tokenizer_decode(self): self.assertListEqual(decoded_tok, decoded_processor) def test_model_input_names(self): - feature_extractor = self.get_feature_extractor() + image_processor = self.get_image_processor() tokenizer = self.get_tokenizer() - processor = BlipProcessor(tokenizer=tokenizer, feature_extractor=feature_extractor) + processor = BlipProcessor(tokenizer=tokenizer, image_processor=image_processor) input_str = "lower newer" image_input = self.prepare_image_inputs() From afa4e1e06877fc32f0d765d2d236a3ca0a938375 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Thu, 15 Dec 2022 15:39:59 +0000 Subject: [PATCH 88/96] fix doctests --- src/transformers/models/blip/modeling_blip.py | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 58ca23401142..f24546ab3db5 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -771,7 +771,7 @@ def get_text_features( >>> model = BlipModel.from_pretrained("Salesforce/blip-image-captioning-base") >>> processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") - >>> inputs = processor(["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") + >>> inputs = processor(text=["a photo of a cat", "a photo of a dog"], padding=True, return_tensors="pt") >>> text_features = model.get_text_features(**inputs) ```""" return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -982,8 +982,7 @@ def forward( >>> inputs = processor(images=image, return_tensors="pt") - >>> outputs = model.generate(**inputs) - >>> print(processor.decode(outputs[0], skip_special_tokens=True)) + >>> outputs = model(**inputs) ```""" batch_size = pixel_values.shape[0] return_dict = return_dict if return_dict is not None else self.config.use_return_dict @@ -1059,6 +1058,8 @@ def generate( >>> inputs = processor(images=image, return_tensors="pt") >>> outputs = model.generate(**inputs) + >>> print(processor.decode(outputs[0], skip_special_tokens=True)) + two cats are laying on a couch ``` """ @@ -1155,10 +1156,11 @@ def forward( >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) + >>> text = "How many cats are in the picture?" - >>> inputs = processor(images=image, return_tensors="pt") + >>> inputs = processor(images=image, text=text, return_tensors="pt") - >>> outputs = model.generate(**inputs) + >>> outputs = model(**inputs) ```""" return_dict = return_dict if return_dict is not None else self.config.use_return_dict batch_size = input_ids.shape[0] @@ -1240,18 +1242,20 @@ def generate( ```python >>> from PIL import Image >>> import requests - >>> from transformers import BlipTokenizer, BlipForImageCaptioning + >>> from transformers import BlipProcessor, BlipForQuestionAnswering - >>> model = BlipForImageCaptioning.from_pretrained("Salesforce/blip-image-captioning-base") - >>> processor = CLIPProcessor.from_pretrained("Salesforce/blip-image-captioning-base") + >>> model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base") + >>> processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) + >>> text = "How many cats are in the picture?" - >>> inputs = processor(images=image, return_tensors="pt") + >>> inputs = processor(images=image, text=text, return_tensors="pt") - >>> outputs = model(**inputs) - >>> image_embeds = outputs.image_embeds + >>> outputs = model.generate(**inputs) + >>> print(processor.decode(outputs[0], skip_special_tokens=True)) + 2 ``` """ vision_outputs = self.vision_model( @@ -1351,18 +1355,17 @@ def forward( >>> import requests >>> from transformers import BlipProcessor, BlipForImageTextRetrieval - >>> model = BlipForImageTextRetrieval.from_pretrained("Salesforce/blip-image-captioning-base") - >>> processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") + >>> model = BlipForImageTextRetrieval.from_pretrained("Salesforce/blip-itm-base") + >>> processor = BlipProcessor.from_pretrained("Salesforce/blip-itm-base") >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> text = "an image of a cat" >>> inputs = processor(images=image, text=text, return_tensors="pt") - >>> outputs = model(**inputs) - >>> print(outputs[0]) # similarity score - ```""" + ``` + """ return_dict = return_dict if return_dict is not None else self.config.use_return_dict vision_outputs = self.vision_model( From fae2b2eb4349f2143b7f036f9093a546f5ada5dd Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Thu, 15 Dec 2022 15:55:58 +0000 Subject: [PATCH 89/96] fix weight init order + add fp16 slow test --- src/transformers/models/blip/modeling_blip.py | 29 +++++++++++-------- tests/models/blip/test_modeling_blip.py | 27 +++++++++++++++++ 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index f24546ab3db5..01d5e329279d 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -223,11 +223,7 @@ def __init__(self, config: BlipVisionConfig): self.patch_size = config.patch_size self.class_embedding = nn.Parameter( - nn.init.trunc_normal_( - torch.zeros(1, 1, self.embed_dim, dtype=torch.float32), - mean=0.0, - std=config.initializer_range, - ) + torch.randn(1, 1, self.embed_dim), ) self.patch_embedding = nn.Conv2d( @@ -237,13 +233,7 @@ def __init__(self, config: BlipVisionConfig): self.num_patches = (self.image_size // self.patch_size) ** 2 self.num_positions = self.num_patches + 1 - self.position_embedding = nn.Parameter( - nn.init.trunc_normal_( - torch.zeros(1, self.num_positions, self.embed_dim, dtype=torch.float32), - mean=0.0, - std=config.initializer_range, - ) - ) + self.position_embedding = nn.Parameter(torch.randn(1, self.num_positions, self.embed_dim)) def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: batch_size = pixel_values.shape[0] @@ -446,6 +436,21 @@ def _init_weights(self, module): if hasattr(module, "bias") and module.bias is not None: module.bias.data.zero_() + if isinstance(module, BlipVisionEmbeddings): + if hasattr(self.config, "vision_config"): + factor = self.config.vision_config.initializer_range + nn.init.trunc_normal_( + module.position_embedding, + mean=0.0, + std=factor, + ) + + nn.init.trunc_normal_( + module.class_embedding, + mean=0.0, + std=factor, + ) + elif isinstance(module, nn.LayerNorm): module.bias.data.zero_() module.weight.data.fill_(1.0) diff --git a/tests/models/blip/test_modeling_blip.py b/tests/models/blip/test_modeling_blip.py index 57c0cae872bf..b74d94edb47a 100644 --- a/tests/models/blip/test_modeling_blip.py +++ b/tests/models/blip/test_modeling_blip.py @@ -801,6 +801,33 @@ def test_inference_image_captioning(self): [30522, 1037, 3861, 1997, 1037, 2450, 3564, 2006, 1996, 3509, 2007, 2014, 3899, 102], ) + def test_inference_image_captioning_fp16(self): + model = BlipForConditionalGeneration.from_pretrained( + "Salesforce/blip-image-captioning-base", torch_dtype=torch.float16 + ).to(torch_device) + processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base") + image = prepare_img() + + # image only + inputs = processor(images=image, return_tensors="pt").to(torch_device, torch.float16) + + predictions = model.generate(**inputs) + + # Test output + self.assertEqual(predictions[0].tolist(), [30522, 1037, 2450, 3564, 2006, 1996, 3509, 2007, 2014, 3899, 102]) + + # image and context + context = ["a picture of"] + inputs = processor(images=image, text=context, return_tensors="pt").to(torch_device, torch.float16) + + predictions = model.generate(**inputs) + + # Test output + self.assertEqual( + predictions[0].tolist(), + [30522, 1037, 3861, 1997, 1037, 2450, 3564, 2006, 1996, 3509, 2007, 2014, 3899, 102], + ) + def test_inference_vqa(self): model = BlipForQuestionAnswering.from_pretrained("Salesforce/blip-vqa-base").to(torch_device) processor = BlipProcessor.from_pretrained("Salesforce/blip-vqa-base") From 2a3b272b2ad64a462461037ba7c0b9eaa3209831 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Thu, 15 Dec 2022 16:06:10 +0000 Subject: [PATCH 90/96] add `blip` to doctest --- utils/documentation_tests.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/documentation_tests.txt b/utils/documentation_tests.txt index 9293dc3c3934..579a44d4df5f 100644 --- a/utils/documentation_tests.txt +++ b/utils/documentation_tests.txt @@ -35,6 +35,7 @@ src/transformers/models/blenderbot/configuration_blenderbot.py src/transformers/models/blenderbot/modeling_blenderbot.py src/transformers/models/blenderbot_small/configuration_blenderbot_small.py src/transformers/models/blenderbot_small/modeling_blenderbot_small.py +src/transformers/models/blip/modeling_blip.py src/transformers/models/bloom/configuration_bloom.py src/transformers/models/camembert/configuration_camembert.py src/transformers/models/canine/configuration_canine.py From f7d7b4324e4b1a6bba05524f5b69053fdcca56c0 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Thu, 15 Dec 2022 18:32:00 +0000 Subject: [PATCH 91/96] add correct repo name and fix test --- .../models/blip/configuration_blip.py | 18 ++++++------------ src/transformers/models/blip/modeling_blip.py | 12 +++++++----- .../models/blip/processing_blip.py | 1 - tests/models/blip/test_modeling_blip.py | 4 ++-- 4 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/transformers/models/blip/configuration_blip.py b/src/transformers/models/blip/configuration_blip.py index 7c337cb6645d..781c15637cc3 100644 --- a/src/transformers/models/blip/configuration_blip.py +++ b/src/transformers/models/blip/configuration_blip.py @@ -26,7 +26,7 @@ BLIP_PRETRAINED_CONFIG_ARCHIVE_MAP = { "Salesforce/blip-vqa-base": "https://huggingface.co/Salesforce/blip-vqa-base/resolve/main/config.json", - "Salesforce/blip-vqa-base-capfit": ( + "Salesforce/blip-vqa-capfit-large": ( "https://huggingface.co/Salesforce/blip-vqa-base-capfit/resolve/main/config.json" ), "Salesforce/blip-image-captioning-base": ( @@ -35,17 +35,11 @@ "Salesforce/blip-image-captioning-large": ( "https://huggingface.co/Salesforce/blip-image-captioning-large/resolve/main/config.json" ), - "Salesforce/blip-retrieval-coco-base": ( - "https://huggingface.co/Salesforce/blip-retrieval-coco-base/resolve/main/config.json" - ), - "Salesforce/blip-retrieval-coco-large": ( - "https://huggingface.co/Salesforce/blip-retrieval-coco-large/resolve/main/config.json" - ), - "Salesforce/blip-retrieval-flikr-base": ( - "https://huggingface.co/Salesforce/blip-retrieval-flikr-base/resolve/main/config.json" - ), - "Salesforce/blip-retrieval-flikr-large": ( - "https://huggingface.co/Salesforce/blip-retrieval-flikr-large/resolve/main/config.json" + "Salesforce/blip-itm-base-coco": "https://huggingface.co/Salesforce/blip-itm-base-coco/resolve/main/config.json", + "Salesforce/blip-itm-large-coco": "https://huggingface.co/Salesforce/blip-itm-large-coco/resolve/main/config.json", + "Salesforce/blip-itm-base-flikr": "https://huggingface.co/Salesforce/blip-itm-base-flikr/resolve/main/config.json", + "Salesforce/blip-itm-large-flikr": ( + "https://huggingface.co/Salesforce/blip-itm-large-flikr/resolve/main/config.json" ), } diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 01d5e329279d..6941cf487c1e 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -42,13 +42,13 @@ BLIP_PRETRAINED_MODEL_ARCHIVE_LIST = [ "Salesforce/blip-vqa-base", - "Salesforce/blip-vqa-base-capfit", + "Salesforce/blip-vqa-capfit-large", "Salesforce/blip-image-captioning-base", "Salesforce/blip-image-captioning-large", - "Salesforce/blip-retrieval-coco-base", - "Salesforce/blip-retrieval-coco-large", - "Salesforce/blip-retrieval-flikr-base", - "Salesforce/blip-retrieval-flikr-large", + "Salesforce/blip-itm-base-coco", + "Salesforce/blip-itm-large-coco", + "Salesforce/blip-itm-base-flikr", + "Salesforce/blip-itm-large-flikr", # See all BLIP models at https://huggingface.co/models?filter=blip ] @@ -658,6 +658,8 @@ class BlipVisionModel(BlipPreTrainedModel): main_input_name = "pixel_values" def __init__(self, config: BlipVisionConfig): + if hasattr(config, "vision_config"): + config = config.vision_config super().__init__(config) self.config = config embed_dim = config.hidden_size diff --git a/src/transformers/models/blip/processing_blip.py b/src/transformers/models/blip/processing_blip.py index 9916299179f4..a4c42ff0a1bf 100644 --- a/src/transformers/models/blip/processing_blip.py +++ b/src/transformers/models/blip/processing_blip.py @@ -96,7 +96,6 @@ def __call__( return_tensors=return_tensors, **kwargs, ) - _ = text_encoding.pop("token_type_ids", None) return text_encoding # add pixel_values + pixel_mask diff --git a/tests/models/blip/test_modeling_blip.py b/tests/models/blip/test_modeling_blip.py index b74d94edb47a..f858f0595cae 100644 --- a/tests/models/blip/test_modeling_blip.py +++ b/tests/models/blip/test_modeling_blip.py @@ -842,8 +842,8 @@ def test_inference_vqa(self): self.assertEqual(out[0].tolist(), [30522, 1015, 102]) def test_inference_itm(self): - model = BlipForImageTextRetrieval.from_pretrained("Salesforce/blip-itm-base").to(torch_device) - processor = BlipProcessor.from_pretrained("Salesforce/blip-itm-base") + model = BlipForImageTextRetrieval.from_pretrained("Salesforce/blip-itm-base-coco").to(torch_device) + processor = BlipProcessor.from_pretrained("Salesforce/blip-itm-base-coco") image = prepare_img() text = "A woman and her dog sitting in a beach" From 4daa6c866ea62aa80ba58aa4be6025f16741b638 Mon Sep 17 00:00:00 2001 From: Younes Belkada <49240599+younesbelkada@users.noreply.github.com> Date: Thu, 15 Dec 2022 21:08:56 +0100 Subject: [PATCH 92/96] Update src/transformers/models/blip/processing_blip.py Co-authored-by: NielsRogge <48327001+NielsRogge@users.noreply.github.com> --- src/transformers/models/blip/processing_blip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/blip/processing_blip.py b/src/transformers/models/blip/processing_blip.py index a4c42ff0a1bf..e860f6723a26 100644 --- a/src/transformers/models/blip/processing_blip.py +++ b/src/transformers/models/blip/processing_blip.py @@ -98,7 +98,7 @@ def __call__( ) return text_encoding - # add pixel_values + pixel_mask + # add pixel_values encoding_image_processor = self.image_processor(images, return_tensors=return_tensors) if text is not None: From 64762b52ba2424c686ac9f94bf6c19bc379aec40 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Thu, 15 Dec 2022 20:24:59 +0000 Subject: [PATCH 93/96] fix tests --- src/transformers/models/blip/modeling_blip.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/transformers/models/blip/modeling_blip.py b/src/transformers/models/blip/modeling_blip.py index 6941cf487c1e..8856fe04e867 100644 --- a/src/transformers/models/blip/modeling_blip.py +++ b/src/transformers/models/blip/modeling_blip.py @@ -656,10 +656,9 @@ def custom_forward(*inputs): class BlipVisionModel(BlipPreTrainedModel): main_input_name = "pixel_values" + config_class = BlipVisionConfig def __init__(self, config: BlipVisionConfig): - if hasattr(config, "vision_config"): - config = config.vision_config super().__init__(config) self.config = config embed_dim = config.hidden_size From aa7396082658e0bda0d236355da66ca9b2970e1a Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Thu, 15 Dec 2022 20:33:44 +0000 Subject: [PATCH 94/96] use `convert_to_rgb` from `image_transforms` --- .../models/blip/image_processing_blip.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/src/transformers/models/blip/image_processing_blip.py b/src/transformers/models/blip/image_processing_blip.py index a1152bc7f10e..810a2a1e4c15 100644 --- a/src/transformers/models/blip/image_processing_blip.py +++ b/src/transformers/models/blip/image_processing_blip.py @@ -22,7 +22,7 @@ from transformers.utils.generic import TensorType from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict -from ...image_transforms import normalize, rescale, resize, to_channel_dimension_format +from ...image_transforms import convert_to_rgb, normalize, rescale, resize, to_channel_dimension_format from ...image_utils import ( IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD, @@ -43,21 +43,6 @@ logger = logging.get_logger(__name__) -# Copied from transformers.models.clip.image_processing_clip.convert_to_rgb -def convert_to_rgb(image: Union[Any, PIL.Image.Image]) -> Union[Any, PIL.Image.Image]: - """ - Converts `PIL.Image.Image` to RGB format. Images in other formats are returned as is. - - Args: - image (`PIL.Image.Image`): - The image to convert. - """ - if not isinstance(image, PIL.Image.Image): - return image - - return image.convert("RGB") - - class BlipImageProcessor(BaseImageProcessor): r""" Constructs a BLIP image processor. From 9888c9b2b25798eda8bd1ff8fd6cafa8be6f6783 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Thu, 15 Dec 2022 20:34:34 +0000 Subject: [PATCH 95/96] make fixup --- src/transformers/models/blip/image_processing_blip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/blip/image_processing_blip.py b/src/transformers/models/blip/image_processing_blip.py index 810a2a1e4c15..4310a073fcad 100644 --- a/src/transformers/models/blip/image_processing_blip.py +++ b/src/transformers/models/blip/image_processing_blip.py @@ -14,7 +14,7 @@ # limitations under the License. """Image processor class for BLIP.""" -from typing import Any, Dict, List, Optional, Union +from typing import Dict, List, Optional, Union import numpy as np From a77b52c1748abc9d32db35ca15ea3880eda519c6 Mon Sep 17 00:00:00 2001 From: younesbelkada Date: Fri, 16 Dec 2022 08:52:25 +0000 Subject: [PATCH 96/96] fix large loading issue --- src/transformers/models/blip/configuration_blip.py | 6 ++++++ src/transformers/models/blip/modeling_blip_text.py | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/transformers/models/blip/configuration_blip.py b/src/transformers/models/blip/configuration_blip.py index 781c15637cc3..3ed32824d09a 100644 --- a/src/transformers/models/blip/configuration_blip.py +++ b/src/transformers/models/blip/configuration_blip.py @@ -61,6 +61,8 @@ class BlipTextConfig(PretrainedConfig): the `inputs_ids` passed when calling [`BlipModel`]. hidden_size (`int`, *optional*, defaults to 768): Dimensionality of the encoder layers and the pooler layer. + encoder_hidden_size (`int`, *optional*, defaults to 768): + Dimensionality of the encoder layers from the vision model. intermediate_size (`int`, *optional*, defaults to 3072): Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder. num_hidden_layers (`int`, *optional*, defaults to 12): @@ -116,6 +118,7 @@ def __init__( self, vocab_size=30524, hidden_size=768, + encoder_hidden_size=768, intermediate_size=3072, projection_dim=768, num_hidden_layers=12, @@ -145,6 +148,7 @@ def __init__( self.vocab_size = vocab_size self.hidden_size = hidden_size + self.encoder_hidden_size = encoder_hidden_size self.intermediate_size = intermediate_size self.projection_dim = projection_dim self.hidden_dropout_prob = hidden_dropout_prob @@ -365,6 +369,8 @@ def __init__( self.text_config = BlipTextConfig(**text_config) self.vision_config = BlipVisionConfig(**vision_config) + self.text_config.encoder_hidden_size = self.vision_config.hidden_size + self.projection_dim = projection_dim self.logit_scale_init_value = logit_scale_init_value self.initializer_factor = 1.0 diff --git a/src/transformers/models/blip/modeling_blip_text.py b/src/transformers/models/blip/modeling_blip_text.py index c0146a631ac4..a72012ef2c00 100644 --- a/src/transformers/models/blip/modeling_blip_text.py +++ b/src/transformers/models/blip/modeling_blip_text.py @@ -104,8 +104,8 @@ def __init__(self, config, is_cross_attention): self.query = nn.Linear(config.hidden_size, self.all_head_size) if is_cross_attention: - self.key = nn.Linear(config.hidden_size, self.all_head_size) - self.value = nn.Linear(config.hidden_size, self.all_head_size) + self.key = nn.Linear(config.encoder_hidden_size, self.all_head_size) + self.value = nn.Linear(config.encoder_hidden_size, self.all_head_size) else: self.key = nn.Linear(config.hidden_size, self.all_head_size) self.value = nn.Linear(config.hidden_size, self.all_head_size)