Assertion error while loading the model
#1
by
b0yan9
- opened
Hi,
I tried to use this model but there's an assertion error when loading the model
assert param_data.shape == loaded_weight.shape
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
My command is vllm serve unsloth/Qwen2.5-VL-32B-Instruct-unsloth-bnb-4bit --dtype bfloat16 --load_format bitsandbytes --quantization bitsandbytes
The full exception output is below
Traceback (most recent call last):
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/engine/multiprocessing/engine.py", line 436, in run_mp_engine
engine = MQLLMEngine.from_vllm_config(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/engine/multiprocessing/engine.py", line 128, in from_vllm_config
return cls(
^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/engine/multiprocessing/engine.py", line 82, in __init__
self.engine = LLMEngine(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/engine/llm_engine.py", line 280, in __init__
self.model_executor = executor_class(vllm_config=vllm_config, )
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/executor/executor_base.py", line 52, in __init__
self._init_executor()
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/executor/uniproc_executor.py", line 47, in _init_executor
self.collective_rpc("load_model")
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/executor/uniproc_executor.py", line 56, in collective_rpc
answer = run_method(self.driver_worker, method, args, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/utils.py", line 2255, in run_method
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/worker/worker.py", line 183, in load_model
self.model_runner.load_model()
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/worker/model_runner.py", line 1113, in load_model
self.model = get_model(vllm_config=self.vllm_config)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/model_loader/__init__.py", line 14, in get_model
return loader.load_model(vllm_config=vllm_config)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/model_loader/loader.py", line 1278, in load_model
self._load_weights(model_config, model)
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/model_loader/loader.py", line 1188, in _load_weights
loaded_weights = model.load_weights(qweight_iterator)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/qwen2_5_vl.py", line 1113, in load_weights
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/utils.py", line 235, in load_weights
autoloaded_weights = set(self._load_module("", self.module, weights))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/utils.py", line 196, in _load_module
yield from self._load_module(prefix,
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/utils.py", line 173, in _load_module
loaded_params = module_load_weights(weights)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/qwen2.py", line 490, in load_weights
return loader.load_weights(weights)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/utils.py", line 235, in load_weights
autoloaded_weights = set(self._load_module("", self.module, weights))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/utils.py", line 196, in _load_module
yield from self._load_module(prefix,
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/utils.py", line 173, in _load_module
loaded_params = module_load_weights(weights)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/qwen2.py", line 403, in load_weights
weight_loader(param, loaded_weight)
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/layers/linear.py", line 1228, in weight_loader
assert param_data.shape == loaded_weight.shape
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError
Process SpawnProcess-1:
Traceback (most recent call last):
File "/home/bo/.conda/envs/vllm/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap
self.run()
File "/home/bo/.conda/envs/vllm/lib/python3.12/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/engine/multiprocessing/engine.py", line 450, in run_mp_engine
raise e
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/engine/multiprocessing/engine.py", line 436, in run_mp_engine
engine = MQLLMEngine.from_vllm_config(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/engine/multiprocessing/engine.py", line 128, in from_vllm_config
return cls(
^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/engine/multiprocessing/engine.py", line 82, in __init__
self.engine = LLMEngine(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/engine/llm_engine.py", line 280, in __init__
self.model_executor = executor_class(vllm_config=vllm_config, )
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/executor/executor_base.py", line 52, in __init__
self._init_executor()
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/executor/uniproc_executor.py", line 47, in _init_executor
self.collective_rpc("load_model")
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/executor/uniproc_executor.py", line 56, in collective_rpc
answer = run_method(self.driver_worker, method, args, kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/utils.py", line 2255, in run_method
return func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/worker/worker.py", line 183, in load_model
self.model_runner.load_model()
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/worker/model_runner.py", line 1113, in load_model
self.model = get_model(vllm_config=self.vllm_config)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/model_loader/__init__.py", line 14, in get_model
return loader.load_model(vllm_config=vllm_config)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/model_loader/loader.py", line 1278, in load_model
self._load_weights(model_config, model)
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/model_loader/loader.py", line 1188, in _load_weights
loaded_weights = model.load_weights(qweight_iterator)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/qwen2_5_vl.py", line 1113, in load_weights
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/utils.py", line 235, in load_weights
autoloaded_weights = set(self._load_module("", self.module, weights))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/utils.py", line 196, in _load_module
yield from self._load_module(prefix,
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/utils.py", line 173, in _load_module
loaded_params = module_load_weights(weights)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/qwen2.py", line 490, in load_weights
return loader.load_weights(weights)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/utils.py", line 235, in load_weights
autoloaded_weights = set(self._load_module("", self.module, weights))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/utils.py", line 196, in _load_module
yield from self._load_module(prefix,
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/utils.py", line 173, in _load_module
loaded_params = module_load_weights(weights)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/models/qwen2.py", line 403, in load_weights
weight_loader(param, loaded_weight)
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/model_executor/layers/linear.py", line 1228, in weight_loader
assert param_data.shape == loaded_weight.shape
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError
Loading safetensors checkpoint shards: 20% Completed | 1/5 [00:01<00:06, 1.59s/it]
Traceback (most recent call last):
File "/home/bo/.conda/envs/vllm/bin/vllm", line 8, in <module>
sys.exit(main())
^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/entrypoints/cli/main.py", line 75, in main
args.dispatch_function(args)
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/entrypoints/cli/serve.py", line 33, in cmd
uvloop.run(run_server(args))
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/uvloop/__init__.py", line 109, in run
return __asyncio.run(
^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/asyncio/runners.py", line 195, in run
return runner.run(main)
^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/asyncio/runners.py", line 118, in run
return self._loop.run_until_complete(task)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/uvloop/__init__.py", line 61, in wrapper
return await main
^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 1016, in run_server
async with build_async_engine_client(args) as engine_client:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/contextlib.py", line 210, in __aenter__
return await anext(self.gen)
^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 141, in build_async_engine_client
async with build_async_engine_client_from_engine_args(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/contextlib.py", line 210, in __aenter__
return await anext(self.gen)
^^^^^^^^^^^^^^^^^^^^^
File "/home/bo/.conda/envs/vllm/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 264, in build_async_engine_client_from_engine_args
raise RuntimeError(
RuntimeError: Engine process failed to start. See stack trace for the root cause.```
Same issue here. Using the same command to launch Qwen2.5-VL-7B works fine.
same