wassname/vllm
1
0
Fork 0
mirror of https://github.com/wassname/vllm.git synced 2026-09-12 13:01:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Model] Add AWQ quantization support for InternVL2 model (#7187)

Browse Source
This commit is contained in:
Isotr0py
2024-08-20 23:18:57 -07:00
committed by GitHub
parent b74a125800
commit 12e1c65bc9
4 changed files with 123 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vllm/model_executor/model_loader/weight_utils.py
+4
View File
@@ -131,6 +131,10 @@ def get_quant_config(model_config: ModelConfig,
# Read the quantization config from the HF model config, if available.
hf_quant_config = getattr(model_config.hf_config, "quantization_config",
None)
# some vision model may keep quantization_config in their text_config
hf_text_config = getattr(model_config.hf_config, "text_config", None)
if hf_quant_config is None and hf_text_config is not None:
hf_quant_config = getattr(hf_text_config, "quantization_config", None)
if hf_quant_config is None:
# compressed-tensors uses a compressions_config
hf_quant_config = getattr(model_config.hf_config, "compression_config",