add qwen2vl for sequence classification

src/transformers/__init__.py

@@ -3119,6 +3119,7 @@
     _import_structure["models.qwen2_vl"].extend(
         [
             "Qwen2VLForConditionalGeneration",
+            "Qwen2VLForSequenceClassification",
             "Qwen2VLModel",
             "Qwen2VLPreTrainedModel",
         ]
@@ -7644,6 +7645,7 @@
         )
         from .models.qwen2_vl import (
             Qwen2VLForConditionalGeneration,
+            Qwen2VLForSequenceClassification,
             Qwen2VLModel,
             Qwen2VLPreTrainedModel,
         )

src/transformers/models/qwen2_vl/__init__.py

@@ -30,6 +30,7 @@
 else:
     _import_structure["modeling_qwen2_vl"] = [
         "Qwen2VLForConditionalGeneration",
+        "Qwen2VLForSequenceClassification",
         "Qwen2VLModel",
         "Qwen2VLPreTrainedModel",
     ]
@@ -55,6 +56,7 @@
     else:
         from .modeling_qwen2_vl import (
             Qwen2VLForConditionalGeneration,
+            Qwen2VLForSequenceClassification,
             Qwen2VLModel,
             Qwen2VLPreTrainedModel,
         )

src/transformers/models/qwen2_vl/modeling_qwen2_vl.py

@@ -27,7 +27,7 @@
 import torch.nn as nn
 import torch.nn.functional as F
 import torch.utils.checkpoint
-from torch.nn import CrossEntropyLoss, LayerNorm
+from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, LayerNorm, MSELoss
 
 from ...activations import ACT2FN
 from ...cache_utils import Cache, SlidingWindowCache, StaticCache
@@ -38,6 +38,7 @@
 from ...modeling_outputs import (
     BaseModelOutputWithPast,
     ModelOutput,
+    SequenceClassifierOutputWithPast,
 )
 from ...modeling_rope_utils import ROPE_INIT_FUNCTIONS
 from ...modeling_utils import PreTrainedModel
@@ -1860,3 +1861,173 @@ def prepare_inputs_for_generation(
             }
         )
         return model_inputs
+
+
+@add_start_docstrings(
+    """(Inspired by the LlamaForSequenceClassification)
+    The Qwen2-VL transformer with a sequence classification head on top (linear layer).
+    Copied the Qwen2VLForConditionalGenerations forward method, but with modification such that it supports sequence classification using the last token embedding instead.
+
+    [`Qwen2VLForSequenceClassification`] uses the last token in order to do the classification, as other causal models
+    (e.g. GPT-2) do.
+
+    Since it does classification on the last token, it requires to know the position of the last token. If a
+    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
+    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
+    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
+    each row of the batch).
+    """,
+    QWEN2VL_START_DOCSTRING,
+)
+class Qwen2VLForSequenceClassification(Qwen2VLPreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        self.num_labels = config.num_labels
+        self.visual = Qwen2VisionTransformerPretrainedModel._from_config(
+            config.vision_config, attn_implementation=config._attn_implementation
+        )
+        self.model = Qwen2VLModel(config)
+        self.vocab_size = config.vocab_size
+        self.classification_head = nn.Linear(config.hidden_size, self.num_labels, bias=False)
+        self.padding_side = "left"  # set it to left by default, user can use setter to change padding_sides
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.model.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.model.embed_tokens = value
+
+    @add_start_docstrings_to_model_forward(QWEN2_VL_INPUTS_DOCSTRING)
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        pixel_values: Optional[torch.Tensor] = None,
+        pixel_values_videos: Optional[torch.FloatTensor] = None,
+        image_grid_thw: Optional[torch.LongTensor] = None,
+        video_grid_thw: Optional[torch.LongTensor] = None,
+        rope_deltas: Optional[torch.LongTensor] = None,
+    ) -> Union[Tuple, SequenceClassifierOutputWithPast]:
+        r"""(copied from LlamaForSequenceClassification)
+        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+        """
+
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        if inputs_embeds is None:
+            inputs_embeds = self.model.embed_tokens(input_ids)
+            if pixel_values is not None:
+                pixel_values = pixel_values.type(self.visual.get_dtype())
+                image_embeds = self.visual(pixel_values, grid_thw=image_grid_thw)
+                image_mask = (
+                    (input_ids == self.config.image_token_id)
+                    .unsqueeze(-1)
+                    .expand_as(inputs_embeds)
+                    .to(inputs_embeds.device)
+                )
+                image_embeds = image_embeds.to(inputs_embeds.device, inputs_embeds.dtype)
+                inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds)
+
+            if pixel_values_videos is not None:
+                pixel_values_videos = pixel_values_videos.type(self.visual.get_dtype())
+                video_embeds = self.visual(pixel_values_videos, grid_thw=video_grid_thw)
+                video_mask = (
+                    (input_ids == self.config.video_token_id)
+                    .unsqueeze(-1)
+                    .expand_as(inputs_embeds)
+                    .to(inputs_embeds.device)
+                )
+                video_embeds = video_embeds.to(inputs_embeds.device, inputs_embeds.dtype)
+                inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds)
+
+            if attention_mask is not None:
+                attention_mask = attention_mask.to(inputs_embeds.device)
+
+        transformer_outputs = self.model(
+            input_ids=None,
+            position_ids=position_ids,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        hidden_states: torch.Tensor = transformer_outputs[0]
+        logits: torch.Tensor = self.classification_head(hidden_states)
+
+        # Below copied from LlamaForSequenceClassification code
+        if input_ids is not None:
+            batch_size = input_ids.shape[0]
+        else:
+            batch_size = inputs_embeds.shape[0]
+
+        if self.config.pad_token_id is None and batch_size != 1:
+            raise ValueError("Cannot handle batch sizes > 1 if no padding token is defined.")
+        if self.config.pad_token_id is None:
+            sequence_lengths = -1
+        else:
+            if input_ids is not None:
+                # if no pad token found, use modulo instead of reverse indexing for ONNX compatibility
+                sequence_lengths = torch.eq(input_ids, self.config.pad_token_id).int().argmax(-1) - 1
+                sequence_lengths = sequence_lengths % input_ids.shape[-1]
+                sequence_lengths = sequence_lengths.to(logits.device)
+            else:
+                sequence_lengths = -1
+
+        pooled_logits = logits[torch.arange(batch_size, device=logits.device), sequence_lengths]
+
+        loss = None
+        if labels is not None:
+            labels = labels.to(logits.device)
+            if self.config.problem_type is None:
+                if self.num_labels == 1:
+                    self.config.problem_type = "regression"
+                elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
+                    self.config.problem_type = "single_label_classification"
+                else:
+                    self.config.problem_type = "multi_label_classification"
+
+            if self.config.problem_type == "regression":
+                loss_fct = MSELoss()
+                if self.num_labels == 1:
+                    loss = loss_fct(pooled_logits.squeeze(), labels.squeeze())
+                else:
+                    loss = loss_fct(pooled_logits, labels)
+            elif self.config.problem_type == "single_label_classification":
+                loss_fct = CrossEntropyLoss()
+                loss = loss_fct(pooled_logits.view(-1, self.num_labels), labels.view(-1))
+            elif self.config.problem_type == "multi_label_classification":
+                loss_fct = BCEWithLogitsLoss()
+                loss = loss_fct(pooled_logits, labels)
+        if not return_dict:
+            output = (pooled_logits,) + transformer_outputs[1:]
+            return ((loss,) + output) if loss is not None else output
+
+        return SequenceClassifierOutputWithPast(
+            loss=loss,
+            logits=pooled_logits,
+            past_key_values=transformer_outputs.past_key_values,
+            hidden_states=transformer_outputs.hidden_states,
+            attentions=transformer_outputs.attentions,
+        )

src/transformers/utils/dummy_pt_objects.py

@@ -7497,6 +7497,13 @@ def __init__(self, *args, **kwargs):
         requires_backends(self, ["torch"])
 
 
+class Qwen2VLForSequenceClassification(metaclass=DummyObject):
+    _backends = ["torch"]
+
+    def __init__(self, *args, **kwargs):
+        requires_backends(self, ["torch"])
+
+
 class Qwen2VLModel(metaclass=DummyObject):
     _backends = ["torch"]
 

tests/models/qwen2_vl/test_modeling_qwen2_vl.py

@@ -26,6 +26,7 @@
     is_torch_available,
     is_vision_available,
 )
+from transformers.models.qwen2_vl.modeling_qwen2_vl import Qwen2VLForSequenceClassification
 from transformers.testing_utils import (
     require_flash_attn,
     require_torch,
@@ -222,7 +223,9 @@ class Qwen2VLModelTest(ModelTesterMixin, GenerationTesterMixin, unittest.TestCas
     Model tester for `Qwen2VLForConditionalGeneration`.
     """
 
-    all_model_classes = (Qwen2VLForConditionalGeneration,) if is_torch_available() else ()
+    all_model_classes = (
+        (Qwen2VLForConditionalGeneration, Qwen2VLForSequenceClassification) if is_torch_available() else ()
+    )
     all_generative_model_classes = (Qwen2VLForConditionalGeneration,) if is_torch_available() else ()
     test_pruning = False
     test_head_masking = False
@@ -315,6 +318,42 @@ def test_beam_search_low_memory(self):
     def test_generate_from_inputs_embeds_with_static_cache(self):
         pass
 
+    @unittest.skip("LM test not for VLM")
+    def test_attention_outputs(self):
+        pass
+
+    @unittest.skip("LM test not for VLM")
+    def test_batching_equivalence(self):
+        pass
+
+    @unittest.skip("LM test not for VLM")
+    def test_determinism(self):
+        pass
+
+    @unittest.skip("LM test not for VLM")
+    def test_hidden_states_output(self):
+        pass
+
+    @unittest.skip("LM test not for VLM")
+    def test_inputs_embeds(self):
+        pass
+
+    @unittest.skip("LM test not for VLM")
+    def test_model_outputs_equivalence(self):
+        pass
+
+    @unittest.skip("LM test not for VLM")
+    def test_resize_tokens_embeddings(self):
+        pass
+
+    @unittest.skip("LM test not for VLM")
+    def test_save_load(self):
+        pass
+
+    @unittest.skip("LM test not for VLM")
+    def test_training(self):
+        pass
+
 
 @require_torch
 class Qwen2VLIntegrationTest(unittest.TestCase):
@@ -510,3 +549,68 @@ def test_small_model_integration_test_batch_wo_image_flashatt2(self):
             self.processor.batch_decode(output, skip_special_tokens=True),
             EXPECTED_DECODED_TEXT,
         )
+
+    @slow
+    def test_sequence_classification_multi_class(self):
+        num_labels = 3
+        model = Qwen2VLForSequenceClassification.from_pretrained(
+            "Qwen/Qwen2-VL-2B-Instruct",
+            torch_dtype="auto",
+            device_map="auto",
+            num_labels=num_labels,
+            pad_token_id=-100,
+        )
+        model.to(torch_device)
+        model.eval()
+        text = self.processor.apply_chat_template(self.messages, tokenize=False, add_generation_prompt=True)
+        inputs = self.processor(
+            text=[text, text], images=[self.image, self.image], padding=True, return_tensors="pt"
+        ).to(torch_device)
+        batch_size = inputs["input_ids"].shape[0]
+        labels = torch.eye(n=num_labels, device=torch_device)[torch.randint(low=0, high=3, size=(batch_size,))]
+        result = model(**inputs, labels=labels)
+        self.assertEqual(result.logits.shape, (batch_size, num_labels))
+
+    @slow
+    def test_sequence_classification_single_label(self):
+        num_labels = 1
+        model = Qwen2VLForSequenceClassification.from_pretrained(
+            "Qwen/Qwen2-VL-2B-Instruct",
+            torch_dtype="auto",
+            device_map="auto",
+            num_labels=num_labels,
+            problem_type="single_label_classification",
+            pad_token_id=-100,
+        )
+        model.to(torch_device)
+        model.eval()
+        text = self.processor.apply_chat_template(self.messages, tokenize=False, add_generation_prompt=True)
+        inputs = self.processor(
+            text=[text, text], images=[self.image, self.image], padding=True, return_tensors="pt"
+        ).to(torch_device)
+        batch_size = inputs["input_ids"].shape[0]
+        labels = torch.randint(low=0, high=2, size=(batch_size, num_labels), device=torch_device)
+        result = model(**inputs, labels=labels)
+        self.assertEqual(result.logits.shape, (batch_size, num_labels))
+
+    @slow
+    def test_sequence_classification_multi_label(self):
+        num_labels = 3
+        model = Qwen2VLForSequenceClassification.from_pretrained(
+            "Qwen/Qwen2-VL-2B-Instruct",
+            torch_dtype="auto",
+            device_map="auto",
+            num_labels=num_labels,
+            problem_type="multi_label_classification",
+            pad_token_id=-100,
+        )
+        model.to(torch_device)
+        model.eval()
+        text = self.processor.apply_chat_template(self.messages, tokenize=False, add_generation_prompt=True)
+        inputs = self.processor(
+            text=[text, text], images=[self.image, self.image], padding=True, return_tensors="pt"
+        ).to(torch_device)
+        batch_size = inputs["input_ids"].shape[0]
+        labels = torch.randint(low=0, high=2, size=(batch_size, num_labels), device=torch_device)
+        result = model(**inputs, labels=labels)
+        self.assertEqual(result.logits.shape, (batch_size, num_labels))