config.json

{
 "_name_or_path": "/data/vicuna/vicuna-7b-v1.5",
 "architectures": [
 "MoDMGMLlamaForCausalLM"
 ],
 "attention_bias": false,
 "attention_dropout": 0.0,
 "bos_token_id": 1,
 "eos_token_id": 2,
 "freeze_mm_mlp_adapter": false,
 "hidden_act": "silu",
 "hidden_size": 4096,
 "image_aspect_ratio": "pad",
 "image_global": true,
 "image_grid": 2,
 "image_grid_pinpoints": null,
 "image_size_aux": 1536,
 "initializer_range": 0.02,
 "intermediate_size": 11008,
 "max_position_embeddings": 4096,
 "mm_hidden_size": 1024,
 "mm_hidden_size_aux": 2880,
 "mm_projector_lr": null,
 "mm_projector_type": "mlp2x_gelu",
 "mm_use_im_patch_token": false,
 "mm_use_im_start_end": false,
 "mm_vision_select_feature": "patch",
 "mm_vision_select_layer": -2,
 "mm_vision_tower": "model_zoo/OpenAI/clip-vit-large-patch14-336",
 "mm_vision_tower_aux": "model_zoo/OpenAI/openclip-convnext-large-d-320-laion2B-s29B-b131K-ft-soup",
 "mod": {
 "capacity_factor": 0.5,
 "mod_enable": true,
 "mod_layers_idx": [
 3,
 4,
 5,
 6,
 7,
 8,
 9,
 10,
 11,
 12,
 13,
 14,
 15,
 16,
 17,
 18,
 19,
 20,
 21,
 22,
 23,
 24,
 25,
 26,
 27,
 28,
 29,
 30
 ],
 "mod_mode": "arank_mod",
 "router_aux_loss_coef": 0.01
 },
 "model_type": "mod_mgm",
 "num_attention_heads": 32,
 "num_hidden_layers": 32,
 "num_key_value_heads": 32,
 "optimize_vision_tower": false,
 "optimize_vision_tower_aux": false,
 "pad_token_id": 0,
 "pretraining_tp": 1,
 "rms_norm_eps": 1e-05,
 "rope_scaling": null,
 "rope_theta": 10000.0,
 "tie_word_embeddings": false,
 "tokenizer_model_max_length": 4096,
 "tokenizer_padding_side": "right",
 "torch_dtype": "bfloat16",
 "transformers_version": "4.36.2",
 "tune_mm_mlp_adapter": false,
 "use_cache": true,
 "use_mm_proj": true,
 "vocab_size": 32000
}