Add support to ImageTextToTextPipeline and save/loading support in Processors

Author: Rocketknight1

src/transformers/processing_utils.py

@@ -488,7 +488,7 @@ class ProcessorMixin(PushToHubMixin):
     """
 
     attributes = ["feature_extractor", "tokenizer"]
-    optional_attributes = ["chat_template", "audio_tokenizer"]
+    optional_attributes = ["chat_template", "audio_tokenizer", "response_schema"]
     optional_call_args: list[str] = []
     # Names need to be attr_class for attr in attributes
     feature_extractor_class = None

tests/utils/test_chat_schema_utils.py

@@ -15,7 +15,7 @@
 import tempfile
 import unittest
 
-from transformers import AutoTokenizer
+from transformers import AutoProcessor, AutoTokenizer
 from transformers.testing_utils import require_jmespath
 from transformers.utils.chat_parsing_utils import recursive_parse
 
@@ -150,13 +150,22 @@
 @require_jmespath
 class ChatSchemaParserTest(unittest.TestCase):
     def test_schema_save_load(self):
+        # Has no schema by default
         tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2")
         tokenizer.response_schema = ernie_schema
         with tempfile.TemporaryDirectory() as tmpdir:
             tokenizer.save_pretrained(tmpdir)
             reloaded_tokenizer = AutoTokenizer.from_pretrained(tmpdir)
         self.assertEqual(reloaded_tokenizer.response_schema, ernie_schema)
 
+        # Has no schema by default
+        processor = AutoProcessor.from_pretrained("hf-internal-testing/tiny-random-Qwen2VLForConditionalGeneration")
+        processor.response_schema = ernie_schema
+        with tempfile.TemporaryDirectory() as tmpdir:
+            processor.save_pretrained(tmpdir)
+            reloaded_processor = AutoProcessor.from_pretrained(tmpdir)
+        self.assertEqual(reloaded_processor.response_schema, ernie_schema)
+
     def test_tokenizer_method(self):
         tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2")
         model_out = '<|START_THINKING|>I should call a tool.<|END_THINKING|><|START_ACTION|>[\n    {"tool_call_id": "0", "tool_name": "simple_tool", "parameters": {"temperature_format": "Celsius"}}\n]<|END_ACTION|><|END_OF_TURN_TOKEN|>'