FEAT: modify language init method

config.json

@@ -1,12 +1,16 @@
 {
   "architectures": [
-    "ZiyaBLIP2ForConditionalGeneration"
+    "ZiyaBlip2ForCausalLM"
   ],
   "assistant_name": "<bot>",
   "human_name": "<human>",
   "initializer_factor": 1.0,
   "initializer_range": 0.02,
   "model_type": "blip-2",
+  "auto_map": {
+    "AutoModel": "modeling_ziya_blip2.ZiyaBlip2ForCausalLM",
+    "AutoModelForCausalLM": "modeling_ziya_blip2.ZiyaBlip2ForCausalLM"
+  },
   "num_query_tokens": 32,
   "prompt_prefix": "",
   "qformer_config": {

modeling_ziya_blip2.py

@@ -11,7 +11,6 @@ from transformers.models.blip_2.modeling_blip_2 import Blip2ForConditionalGenera
 from transformers import (
     Blip2PreTrainedModel,
     Blip2VisionModel,
-    AutoModelForCausalLM,
     Blip2QFormerModel,
     PreTrainedTokenizer,
     PreTrainedModel,
@@ -21,7 +20,7 @@ from transformers import (
 logger = logging.get_logger(__name__)
 
 
-class ZiyaBLIP2ForConditionalGeneration(Blip2PreTrainedModel):
+class ZiyaBlip2ForCausalLM(Blip2PreTrainedModel):
     config_class = Blip2Config
     main_input_name = "pixel_values"
     _keys_to_ignore_on_load_missing = [
@@ -38,11 +37,6 @@ class ZiyaBLIP2ForConditionalGeneration(Blip2PreTrainedModel):
 
         self.language_projection = nn.Linear(
             config.qformer_config.hidden_size, config.text_config.hidden_size)
-        if language_model is None:
-            if config.use_decoder_only_language_model:
-                language_model = AutoModelForCausalLM.from_config(config.text_config)
-            else:
-                raise Exception("not impl")
         self.language_model = language_model
 
         # Initialize weights and apply final processing