(EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] EngineCore failed to start. (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] Traceback (most recent call last): (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] File "/opt/conda/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 937, in run_engine_core (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] engine_core = EngineCoreProc(*args, engine_index=dp_rank, **kwargs) (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] File "/opt/conda/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 691, in __init__ (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] super().__init__( (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] File "/opt/conda/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 105, in __init__ (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] self.model_executor = executor_class(vllm_config) (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] ^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] File "/opt/conda/lib/python3.12/site-packages/vllm/v1/executor/abstract.py", line 101, in __init__ (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] self._init_executor() (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] File "/opt/conda/lib/python3.12/site-packages/vllm/v1/executor/ray_executor.py", line 97, in _init_executor (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] self._init_workers_ray(placement_group) (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] File "/opt/conda/lib/python3.12/site-packages/vllm/v1/executor/ray_executor.py", line 366, in _init_workers_ray (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] self.collective_rpc("init_device") (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] File "/opt/conda/lib/python3.12/site-packages/vllm/v1/executor/ray_executor.py", line 489, in collective_rpc (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] return ray.get(ray_worker_outputs, timeout=timeout) (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] File "/opt/conda/lib/python3.12/site-packages/ray/_private/auto_init_hook.py", line 22, in auto_init_wrapper (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] return fn(*args, **kwargs) (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] ^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] File "/opt/conda/lib/python3.12/site-packages/ray/_private/client_mode_hook.py", line 104, in wrapper (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] return func(*args, **kwargs) (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] ^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] File "/opt/conda/lib/python3.12/site-packages/ray/_private/worker.py", line 2967, in get (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] values, debugger_breakpoint = worker.get_objects( (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] ^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] File "/opt/conda/lib/python3.12/site-packages/ray/_private/worker.py", line 1015, in get_objects (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] raise value.as_instanceof_cause() (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] ray.exceptions.RayTaskError(RuntimeError): ray::RayWorkerWrapper.execute_method() (pid=34801, ip=10.105.113.24, actor_id=ccf31f1c27caee81e5a0e4a801000000, repr=) (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 345, in execute_method (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] raise e (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 334, in execute_method (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] return run_method(self, method, args, kwargs) (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] File "/opt/conda/lib/python3.12/site-packages/vllm/v1/serial_utils.py", line 461, in run_method (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] return func(*args, **kwargs) (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] ^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 326, in init_device (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] self.worker.init_device() # type: ignore (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] ^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/gpu_worker.py", line 210, in init_device (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] current_platform.set_device(self.device) (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] File "/opt/conda/lib/python3.12/site-packages/vllm_metax/platform.py", line 148, in set_device (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] _ = torch.zeros(1, device=device) (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ERROR 06-25 11:33:37 [core.py:946] RuntimeError: CUDA driver error: invalid argument (EngineCore_DP0 pid=34622) Process EngineCore_DP0: (EngineCore_DP0 pid=34622) Traceback (most recent call last): (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap (EngineCore_DP0 pid=34622) self.run() (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/multiprocessing/process.py", line 108, in run (EngineCore_DP0 pid=34622) self._target(*self._args, **self._kwargs) (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 950, in run_engine_core (EngineCore_DP0 pid=34622) raise e (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 937, in run_engine_core (EngineCore_DP0 pid=34622) engine_core = EngineCoreProc(*args, engine_index=dp_rank, **kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 691, in __init__ (EngineCore_DP0 pid=34622) super().__init__( (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 105, in __init__ (EngineCore_DP0 pid=34622) self.model_executor = executor_class(vllm_config) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/executor/abstract.py", line 101, in __init__ (EngineCore_DP0 pid=34622) self._init_executor() (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/executor/ray_executor.py", line 97, in _init_executor (EngineCore_DP0 pid=34622) self._init_workers_ray(placement_group) (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/executor/ray_executor.py", line 366, in _init_workers_ray (EngineCore_DP0 pid=34622) self.collective_rpc("init_device") (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/executor/ray_executor.py", line 489, in collective_rpc (EngineCore_DP0 pid=34622) return ray.get(ray_worker_outputs, timeout=timeout) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/ray/_private/auto_init_hook.py", line 22, in auto_init_wrapper (EngineCore_DP0 pid=34622) return fn(*args, **kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/ray/_private/client_mode_hook.py", line 104, in wrapper (EngineCore_DP0 pid=34622) return func(*args, **kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/ray/_private/worker.py", line 2967, in get (EngineCore_DP0 pid=34622) values, debugger_breakpoint = worker.get_objects( (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/ray/_private/worker.py", line 1015, in get_objects (EngineCore_DP0 pid=34622) raise value.as_instanceof_cause() (EngineCore_DP0 pid=34622) ray.exceptions.RayTaskError(RuntimeError): ray::RayWorkerWrapper.execute_method() (pid=34801, ip=10.105.113.24, actor_id=ccf31f1c27caee81e5a0e4a801000000, repr=) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 345, in execute_method (EngineCore_DP0 pid=34622) raise e (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 334, in execute_method (EngineCore_DP0 pid=34622) return run_method(self, method, args, kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/serial_utils.py", line 461, in run_method (EngineCore_DP0 pid=34622) return func(*args, **kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 326, in init_device (EngineCore_DP0 pid=34622) self.worker.init_device() # type: ignore (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/gpu_worker.py", line 210, in init_device (EngineCore_DP0 pid=34622) current_platform.set_device(self.device) (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm_metax/platform.py", line 148, in set_device (EngineCore_DP0 pid=34622) _ = torch.zeros(1, device=device) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) RuntimeError: CUDA driver error: invalid argument (EngineCore_DP0 pid=34622) 2026-06-25 11:33:37,732 ERROR worker.py:433 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=4884, ip=10.105.113.25, actor_id=970f27fd5498813c26ca8ffb01000000, repr=) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 345, in execute_method (EngineCore_DP0 pid=34622) raise e (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 334, in execute_method (EngineCore_DP0 pid=34622) return run_method(self, method, args, kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/serial_utils.py", line 461, in run_method (EngineCore_DP0 pid=34622) return func(*args, **kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 326, in init_device (EngineCore_DP0 pid=34622) self.worker.init_device() # type: ignore (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/gpu_worker.py", line 210, in init_device (EngineCore_DP0 pid=34622) current_platform.set_device(self.device) (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm_metax/platform.py", line 148, in set_device (EngineCore_DP0 pid=34622) _ = torch.zeros(1, device=device) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) RuntimeError: CUDA driver error: invalid argument (EngineCore_DP0 pid=34622) 2026-06-25 11:33:37,733 ERROR worker.py:433 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=4883, ip=10.105.113.25, actor_id=32323b217443e77b4bfcc3b601000000, repr=) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) INFO 06-25 11:33:37 [ray_executor.py:120] Shutting down Ray distributed executor. If you see error log from logging.cc regarding SIGTERM received, please ignore because this is the expected termination process in Ray. (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 345, in execute_method (EngineCore_DP0 pid=34622) raise e (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 334, in execute_method (EngineCore_DP0 pid=34622) return run_method(self, method, args, kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/serial_utils.py", line 461, in run_method (EngineCore_DP0 pid=34622) return func(*args, **kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 326, in init_device (EngineCore_DP0 pid=34622) self.worker.init_device() # type: ignore (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/gpu_worker.py", line 210, in init_device (EngineCore_DP0 pid=34622) current_platform.set_device(self.device) (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm_metax/platform.py", line 148, in set_device (EngineCore_DP0 pid=34622) _ = torch.zeros(1, device=device) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) RuntimeError: CUDA driver error: invalid argument (EngineCore_DP0 pid=34622) 2026-06-25 11:33:37,735 ERROR worker.py:433 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=4881, ip=10.105.113.25, actor_id=518afc3be93baa02025855b001000000, repr=) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 345, in execute_method (EngineCore_DP0 pid=34622) raise e (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 334, in execute_method (EngineCore_DP0 pid=34622) return run_method(self, method, args, kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/serial_utils.py", line 461, in run_method (EngineCore_DP0 pid=34622) return func(*args, **kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 326, in init_device (EngineCore_DP0 pid=34622) self.worker.init_device() # type: ignore (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/gpu_worker.py", line 210, in init_device (EngineCore_DP0 pid=34622) current_platform.set_device(self.device) (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm_metax/platform.py", line 148, in set_device (EngineCore_DP0 pid=34622) _ = torch.zeros(1, device=device) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) RuntimeError: CUDA driver error: invalid argument (EngineCore_DP0 pid=34622) 2026-06-25 11:33:37,736 ERROR worker.py:433 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=4882, ip=10.105.113.25, actor_id=d561ab41929a317e1169b48101000000, repr=) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 345, in execute_method (EngineCore_DP0 pid=34622) raise e (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 334, in execute_method (EngineCore_DP0 pid=34622) return run_method(self, method, args, kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/serial_utils.py", line 461, in run_method (EngineCore_DP0 pid=34622) return func(*args, **kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 326, in init_device (EngineCore_DP0 pid=34622) self.worker.init_device() # type: ignore (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/gpu_worker.py", line 210, in init_device (EngineCore_DP0 pid=34622) current_platform.set_device(self.device) (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm_metax/platform.py", line 148, in set_device (EngineCore_DP0 pid=34622) _ = torch.zeros(1, device=device) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) RuntimeError: CUDA driver error: invalid argument (EngineCore_DP0 pid=34622) 2026-06-25 11:33:37,743 ERROR worker.py:433 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=4880, ip=10.105.113.25, actor_id=729eb7ca1f48cec38c12f55b01000000, repr=) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 345, in execute_method (EngineCore_DP0 pid=34622) raise e (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 334, in execute_method (EngineCore_DP0 pid=34622) return run_method(self, method, args, kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/serial_utils.py", line 461, in run_method (EngineCore_DP0 pid=34622) return func(*args, **kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 326, in init_device (EngineCore_DP0 pid=34622) self.worker.init_device() # type: ignore (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/gpu_worker.py", line 210, in init_device (EngineCore_DP0 pid=34622) current_platform.set_device(self.device) (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm_metax/platform.py", line 148, in set_device (EngineCore_DP0 pid=34622) _ = torch.zeros(1, device=device) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) RuntimeError: CUDA driver error: invalid argument (EngineCore_DP0 pid=34622) 2026-06-25 11:33:37,745 ERROR worker.py:433 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=4879, ip=10.105.113.25, actor_id=4c8ddb4970b16812e12fb33701000000, repr=) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 345, in execute_method (EngineCore_DP0 pid=34622) raise e (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 334, in execute_method (EngineCore_DP0 pid=34622) return run_method(self, method, args, kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/serial_utils.py", line 461, in run_method (EngineCore_DP0 pid=34622) return func(*args, **kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 326, in init_device (EngineCore_DP0 pid=34622) self.worker.init_device() # type: ignore (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/gpu_worker.py", line 210, in init_device (EngineCore_DP0 pid=34622) current_platform.set_device(self.device) (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm_metax/platform.py", line 148, in set_device (EngineCore_DP0 pid=34622) _ = torch.zeros(1, device=device) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) RuntimeError: CUDA driver error: invalid argument (EngineCore_DP0 pid=34622) 2026-06-25 11:33:37,749 ERROR worker.py:433 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=4877, ip=10.105.113.25, actor_id=ac31e0995974ade13970d28301000000, repr=) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 345, in execute_method (EngineCore_DP0 pid=34622) raise e (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 334, in execute_method (EngineCore_DP0 pid=34622) return run_method(self, method, args, kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/serial_utils.py", line 461, in run_method (EngineCore_DP0 pid=34622) return func(*args, **kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 326, in init_device (EngineCore_DP0 pid=34622) self.worker.init_device() # type: ignore (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/gpu_worker.py", line 210, in init_device (EngineCore_DP0 pid=34622) current_platform.set_device(self.device) (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm_metax/platform.py", line 148, in set_device (EngineCore_DP0 pid=34622) _ = torch.zeros(1, device=device) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) RuntimeError: CUDA driver error: invalid argument (EngineCore_DP0 pid=34622) 2026-06-25 11:33:37,749 ERROR worker.py:433 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=34809, ip=10.105.113.24, actor_id=b070017c0ccd4ef7e58aa79b01000000, repr=) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 345, in execute_method (EngineCore_DP0 pid=34622) raise e (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 334, in execute_method (EngineCore_DP0 pid=34622) return run_method(self, method, args, kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/serial_utils.py", line 461, in run_method (EngineCore_DP0 pid=34622) return func(*args, **kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 326, in init_device (EngineCore_DP0 pid=34622) self.worker.init_device() # type: ignore (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/gpu_worker.py", line 210, in init_device (EngineCore_DP0 pid=34622) current_platform.set_device(self.device) (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm_metax/platform.py", line 148, in set_device (EngineCore_DP0 pid=34622) _ = torch.zeros(1, device=device) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) RuntimeError: CUDA driver error: invalid argument (EngineCore_DP0 pid=34622) 2026-06-25 11:33:37,749 ERROR worker.py:433 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=34800, ip=10.105.113.24, actor_id=95bb7498a113f09c689056ca01000000, repr=) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 345, in execute_method (EngineCore_DP0 pid=34622) raise e (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 334, in execute_method (EngineCore_DP0 pid=34622) return run_method(self, method, args, kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/serial_utils.py", line 461, in run_method (EngineCore_DP0 pid=34622) return func(*args, **kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 326, in init_device (EngineCore_DP0 pid=34622) self.worker.init_device() # type: ignore (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/gpu_worker.py", line 210, in init_device (EngineCore_DP0 pid=34622) current_platform.set_device(self.device) (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm_metax/platform.py", line 148, in set_device (EngineCore_DP0 pid=34622) _ = torch.zeros(1, device=device) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) RuntimeError: CUDA driver error: invalid argument (EngineCore_DP0 pid=34622) 2026-06-25 11:33:37,749 ERROR worker.py:433 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=34793, ip=10.105.113.24, actor_id=d1371e11b2a2d223940cbc9b01000000, repr=) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 345, in execute_method (EngineCore_DP0 pid=34622) raise e (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 334, in execute_method (EngineCore_DP0 pid=34622) return run_method(self, method, args, kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/serial_utils.py", line 461, in run_method (EngineCore_DP0 pid=34622) return func(*args, **kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 326, in init_device (EngineCore_DP0 pid=34622) self.worker.init_device() # type: ignore (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/gpu_worker.py", line 210, in init_device (EngineCore_DP0 pid=34622) current_platform.set_device(self.device) (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm_metax/platform.py", line 148, in set_device (EngineCore_DP0 pid=34622) _ = torch.zeros(1, device=device) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) RuntimeError: CUDA driver error: invalid argument (EngineCore_DP0 pid=34622) 2026-06-25 11:33:37,750 ERROR worker.py:433 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=34794, ip=10.105.113.24, actor_id=1f58523e7d6642ccb4b543fd01000000, repr=) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 345, in execute_method (EngineCore_DP0 pid=34622) raise e (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 334, in execute_method (EngineCore_DP0 pid=34622) return run_method(self, method, args, kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/serial_utils.py", line 461, in run_method (EngineCore_DP0 pid=34622) return func(*args, **kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 326, in init_device (EngineCore_DP0 pid=34622) self.worker.init_device() # type: ignore (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/gpu_worker.py", line 210, in init_device (EngineCore_DP0 pid=34622) current_platform.set_device(self.device) (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm_metax/platform.py", line 148, in set_device (EngineCore_DP0 pid=34622) _ = torch.zeros(1, device=device) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) RuntimeError: CUDA driver error: invalid argument (EngineCore_DP0 pid=34622) 2026-06-25 11:33:37,750 ERROR worker.py:433 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=34797, ip=10.105.113.24, actor_id=d6126893dcb3d5beccf8e25a01000000, repr=) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 345, in execute_method (EngineCore_DP0 pid=34622) raise e (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 334, in execute_method (EngineCore_DP0 pid=34622) return run_method(self, method, args, kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/serial_utils.py", line 461, in run_method (EngineCore_DP0 pid=34622) return func(*args, **kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 326, in init_device (EngineCore_DP0 pid=34622) self.worker.init_device() # type: ignore (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/gpu_worker.py", line 210, in init_device (EngineCore_DP0 pid=34622) current_platform.set_device(self.device) (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm_metax/platform.py", line 148, in set_device (EngineCore_DP0 pid=34622) _ = torch.zeros(1, device=device) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) RuntimeError: CUDA driver error: invalid argument (EngineCore_DP0 pid=34622) 2026-06-25 11:33:37,750 ERROR worker.py:433 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=34799, ip=10.105.113.24, actor_id=9dc6b59059a14f8d469e316c01000000, repr=) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 345, in execute_method (EngineCore_DP0 pid=34622) raise e (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 334, in execute_method (EngineCore_DP0 pid=34622) return run_method(self, method, args, kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/serial_utils.py", line 461, in run_method (EngineCore_DP0 pid=34622) return func(*args, **kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 326, in init_device (EngineCore_DP0 pid=34622) self.worker.init_device() # type: ignore (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/gpu_worker.py", line 210, in init_device (EngineCore_DP0 pid=34622) current_platform.set_device(self.device) (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm_metax/platform.py", line 148, in set_device (EngineCore_DP0 pid=34622) _ = torch.zeros(1, device=device) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) RuntimeError: CUDA driver error: invalid argument (EngineCore_DP0 pid=34622) 2026-06-25 11:33:37,750 ERROR worker.py:433 -- Unhandled error (suppress with 'RAY_IGNORE_UNHANDLED_ERRORS=1'): ray::RayWorkerWrapper.execute_method() (pid=34796, ip=10.105.113.24, actor_id=5e5e285416c2707e94933d6c01000000, repr=) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 345, in execute_method (EngineCore_DP0 pid=34622) raise e (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 334, in execute_method (EngineCore_DP0 pid=34622) return run_method(self, method, args, kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/serial_utils.py", line 461, in run_method (EngineCore_DP0 pid=34622) return func(*args, **kwargs) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 326, in init_device (EngineCore_DP0 pid=34622) self.worker.init_device() # type: ignore (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/gpu_worker.py", line 210, in init_device (EngineCore_DP0 pid=34622) current_platform.set_device(self.device) (EngineCore_DP0 pid=34622) File "/opt/conda/lib/python3.12/site-packages/vllm_metax/platform.py", line 148, in set_device (EngineCore_DP0 pid=34622) _ = torch.zeros(1, device=device) (EngineCore_DP0 pid=34622) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) RuntimeError: CUDA driver error: invalid argument (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) WARNING 06-25 11:33:37 [system_utils.py:37] Overwriting environment variable LD_LIBRARY_PATH from '/opt/conda/lib/python3.12/site-packages/cv2/../../lib64:/opt/maca/mxshmem/lib:/opt/maca/mxshmem/lib:/opt/maca/mxshmem/lib:/opt/maca/lib:/opt/maca/mxshmem/lib:/opt/maca/ompi/lib:/opt/maca/ucx/lib:/opt/mxdriver/lib:' to '/opt/conda/lib/python3.12/site-packages/cv2/../../lib64:/opt/conda/lib/python3.12/site-packages/cv2/../../lib64:/opt/maca/mxshmem/lib:/opt/maca/mxshmem/lib:/opt/maca/mxshmem/lib:/opt/maca/lib:/opt/maca/mxshmem/lib:/opt/maca/ompi/lib:/opt/maca/ucx/lib:/opt/mxdriver/lib:' (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) WARNING 06-25 11:33:37 [registry.py:812] Model architecture DeepSeekMTPModel is already registered, and will be overwritten by the new model class vllm_metax.models.deepseek_mtp:DeepSeekMTP. (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) WARNING 06-25 11:33:37 [registry.py:812] Model architecture DeepseekV2ForCausalLM is already registered, and will be overwritten by the new model class vllm_metax.models.deepseek_v2:DeepseekV2ForCausalLM. (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) WARNING 06-25 11:33:37 [registry.py:812] Model architecture DeepseekV3ForCausalLM is already registered, and will be overwritten by the new model class vllm_metax.models.deepseek_v2:DeepseekV3ForCausalLM. (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) WARNING 06-25 11:33:37 [registry.py:812] Model architecture DeepseekV32ForCausalLM is already registered, and will be overwritten by the new model class vllm_metax.models.deepseek_v2:DeepseekV3ForCausalLM. (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) WARNING 06-25 11:33:37 [registry.py:812] Model architecture KimiK25ForConditionalGeneration is already registered, and will be overwritten by the new model class vllm_metax.models.kimi_k25:KimiK25ForConditionalGeneration. (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) WARNING 06-25 11:33:37 [worker_base.py:301] Missing `shared_worker_lock` argument from executor. This argument is needed for mm_processor_cache_type='shm'. (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) ERROR 06-25 11:33:37 [worker_base.py:344] Error executing method 'init_device'. This might cause deadlock in distributed execution. (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) ERROR 06-25 11:33:37 [worker_base.py:344] Traceback (most recent call last): (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) ERROR 06-25 11:33:37 [worker_base.py:344] File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 334, in execute_method (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) ERROR 06-25 11:33:37 [worker_base.py:344] return run_method(self, method, args, kwargs) (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) ERROR 06-25 11:33:37 [worker_base.py:344] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) ERROR 06-25 11:33:37 [worker_base.py:344] File "/opt/conda/lib/python3.12/site-packages/vllm/v1/serial_utils.py", line 461, in run_method (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) ERROR 06-25 11:33:37 [worker_base.py:344] return func(*args, **kwargs) (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) ERROR 06-25 11:33:37 [worker_base.py:344] ^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) ERROR 06-25 11:33:37 [worker_base.py:344] File "/opt/conda/lib/python3.12/site-packages/ray/util/tracing/tracing_helper.py", line 461, in _resume_span (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) ERROR 06-25 11:33:37 [worker_base.py:344] return method(self, *_args, **_kwargs) (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) ERROR 06-25 11:33:37 [worker_base.py:344] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) ERROR 06-25 11:33:37 [worker_base.py:344] File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 326, in init_device (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) ERROR 06-25 11:33:37 [worker_base.py:344] self.worker.init_device() # type: ignore (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) ERROR 06-25 11:33:37 [worker_base.py:344] ^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) ERROR 06-25 11:33:37 [worker_base.py:344] File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/gpu_worker.py", line 210, in init_device (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) ERROR 06-25 11:33:37 [worker_base.py:344] current_platform.set_device(self.device) (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) ERROR 06-25 11:33:37 [worker_base.py:344] File "/opt/conda/lib/python3.12/site-packages/vllm_metax/platform.py", line 148, in set_device (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) ERROR 06-25 11:33:37 [worker_base.py:344] _ = torch.zeros(1, device=device) (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) ERROR 06-25 11:33:37 [worker_base.py:344] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=4877, ip=10.105.113.25) ERROR 06-25 11:33:37 [worker_base.py:344] RuntimeError: CUDA driver error: invalid argument (EngineCore_DP0 pid=34622) (pid=4880, ip=10.105.113.25) INFO Print the version information of mcoplib during compilation. [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (pid=4880, ip=10.105.113.25) Version info:Mcoplib_Version = '0.4.0' [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (pid=4880, ip=10.105.113.25) Build_Maca_Version = '20260226.1027' [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (pid=4880, ip=10.105.113.25) GIT_BRANCH = 'HEAD' [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (pid=4880, ip=10.105.113.25) GIT_COMMIT = 'fe3a7e2' [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (pid=4880, ip=10.105.113.25) Vllm Op Version = 0.15.0 [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (pid=4880, ip=10.105.113.25) SGlang Op Version = 0.5.7 && 0.5.8 [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (pid=4880, ip=10.105.113.25) INFO Staring Check the current MACA version of the operating environment. [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (pid=4880, ip=10.105.113.25) INFO: maca master version matching successful. [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (pid=4880, ip=10.105.113.25) WARNING 06-25 11:33:27 [__init__.py:86] The quantization method 'awq_marlin' already exists and will be overwritten by the quantization config . [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (pid=4884, ip=10.105.113.25) WARNING 06-25 11:33:27 [__init__.py:86] The quantization method 'compressed-tensors' already exists and will be overwritten by the quantization config . [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (pid=4884, ip=10.105.113.25) WARNING 06-25 11:33:27 [__init__.py:86] The quantization method 'gptq' already exists and will be overwritten by the quantization config . [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (pid=4884, ip=10.105.113.25) WARNING 06-25 11:33:27 [__init__.py:86] The quantization method 'gptq_marlin' already exists and will be overwritten by the quantization config . [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (pid=4884, ip=10.105.113.25) WARNING 06-25 11:33:27 [__init__.py:86] The quantization method 'moe_wna16' already exists and will be overwritten by the quantization config . [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=34793) WARNING 06-25 11:33:37 [system_utils.py:37] Overwriting environment variable LD_LIBRARY_PATH from '/opt/conda/lib/python3.12/site-packages/cv2/../../lib64:/opt/maca/mxshmem/lib:/opt/maca/mxshmem/lib:/opt/maca/mxshmem/lib:/opt/maca/lib:/opt/maca/mxshmem/lib:/opt/maca/ompi/lib:/opt/maca/ucx/lib:/opt/mxdriver/lib:' to '/opt/conda/lib/python3.12/site-packages/cv2/../../lib64:/opt/conda/lib/python3.12/site-packages/cv2/../../lib64:/opt/maca/mxshmem/lib:/opt/maca/mxshmem/lib:/opt/maca/mxshmem/lib:/opt/maca/lib:/opt/maca/mxshmem/lib:/opt/maca/ompi/lib:/opt/maca/ucx/lib:/opt/mxdriver/lib:' [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=34793) WARNING 06-25 11:33:37 [registry.py:812] Model architecture DeepSeekMTPModel is already registered, and will be overwritten by the new model class vllm_metax.models.deepseek_mtp:DeepSeekMTP. [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=34793) WARNING 06-25 11:33:37 [registry.py:812] Model architecture KimiK25ForConditionalGeneration is already registered, and will be overwritten by the new model class vllm_metax.models.kimi_k25:KimiK25ForConditionalGeneration. [repeated 60x across cluster] (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=34793) WARNING 06-25 11:33:37 [worker_base.py:301] Missing `shared_worker_lock` argument from executor. This argument is needed for mm_processor_cache_type='shm'. [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=34793) ERROR 06-25 11:33:37 [worker_base.py:344] Error executing method 'init_device'. This might cause deadlock in distributed execution. [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=34793) ERROR 06-25 11:33:37 [worker_base.py:344] Traceback (most recent call last): [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=34793) ERROR 06-25 11:33:37 [worker_base.py:344] File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 334, in execute_method [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=34793) ERROR 06-25 11:33:37 [worker_base.py:344] return run_method(self, method, args, kwargs) [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=34793) ERROR 06-25 11:33:37 [worker_base.py:344] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=34793) ERROR 06-25 11:33:37 [worker_base.py:344] File "/opt/conda/lib/python3.12/site-packages/vllm/v1/serial_utils.py", line 461, in run_method [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=34793) ERROR 06-25 11:33:37 [worker_base.py:344] return func(*args, **kwargs) [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=34793) ERROR 06-25 11:33:37 [worker_base.py:344] ^^^^^^^^^^^^^^^^^^^^^ [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=34793) ERROR 06-25 11:33:37 [worker_base.py:344] File "/opt/conda/lib/python3.12/site-packages/ray/util/tracing/tracing_helper.py", line 461, in _resume_span [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=34793) ERROR 06-25 11:33:37 [worker_base.py:344] return method(self, *_args, **_kwargs) [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=34793) ERROR 06-25 11:33:37 [worker_base.py:344] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=34793) ERROR 06-25 11:33:37 [worker_base.py:344] File "/opt/conda/lib/python3.12/site-packages/vllm/v1/worker/gpu_worker.py", line 210, in init_device [repeated 30x across cluster] (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=34793) ERROR 06-25 11:33:37 [worker_base.py:344] self.worker.init_device() # type: ignore [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=34793) ERROR 06-25 11:33:37 [worker_base.py:344] ^^^^^^^^^^^^^^^^^^^^^^^^^ [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=34793) ERROR 06-25 11:33:37 [worker_base.py:344] current_platform.set_device(self.device) [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=34793) ERROR 06-25 11:33:37 [worker_base.py:344] File "/opt/conda/lib/python3.12/site-packages/vllm_metax/platform.py", line 148, in set_device [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=34793) ERROR 06-25 11:33:37 [worker_base.py:344] _ = torch.zeros(1, device=device) [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=34793) ERROR 06-25 11:33:37 [worker_base.py:344] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [repeated 15x across cluster] (EngineCore_DP0 pid=34622) (RayWorkerWrapper pid=34793) ERROR 06-25 11:33:37 [worker_base.py:344] RuntimeError: CUDA driver error: invalid argument [repeated 15x across cluster] (APIServer pid=34481) Traceback (most recent call last): (APIServer pid=34481) File "/opt/conda/bin/vllm", line 8, in (APIServer pid=34481) sys.exit(main()) (APIServer pid=34481) ^^^^^^ (APIServer pid=34481) File "/opt/conda/lib/python3.12/site-packages/vllm/entrypoints/cli/main.py", line 73, in main (APIServer pid=34481) args.dispatch_function(args) (APIServer pid=34481) File "/opt/conda/lib/python3.12/site-packages/vllm/entrypoints/cli/serve.py", line 111, in cmd (APIServer pid=34481) uvloop.run(run_server(args)) (APIServer pid=34481) File "/opt/conda/lib/python3.12/site-packages/uvloop/__init__.py", line 96, in run (APIServer pid=34481) return __asyncio.run( (APIServer pid=34481) ^^^^^^^^^^^^^^ (APIServer pid=34481) File "/opt/conda/lib/python3.12/asyncio/runners.py", line 195, in run (APIServer pid=34481) return runner.run(main) (APIServer pid=34481) ^^^^^^^^^^^^^^^^ (APIServer pid=34481) File "/opt/conda/lib/python3.12/asyncio/runners.py", line 118, in run (APIServer pid=34481) return self._loop.run_until_complete(task) (APIServer pid=34481) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (APIServer pid=34481) File "uvloop/loop.pyx", line 1518, in uvloop.loop.Loop.run_until_complete (APIServer pid=34481) File "/opt/conda/lib/python3.12/site-packages/uvloop/__init__.py", line 48, in wrapper (APIServer pid=34481) return await main (APIServer pid=34481) ^^^^^^^^^^ (APIServer pid=34481) File "/opt/conda/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 919, in run_server (APIServer pid=34481) await run_server_worker(listen_address, sock, args, **uvicorn_kwargs) (APIServer pid=34481) File "/opt/conda/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 938, in run_server_worker (APIServer pid=34481) async with build_async_engine_client( (APIServer pid=34481) ^^^^^^^^^^^^^^^^^^^^^^^^^^ (APIServer pid=34481) File "/opt/conda/lib/python3.12/contextlib.py", line 210, in __aenter__ (APIServer pid=34481) return await anext(self.gen) (APIServer pid=34481) ^^^^^^^^^^^^^^^^^^^^^ (APIServer pid=34481) File "/opt/conda/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 147, in build_async_engine_client (APIServer pid=34481) async with build_async_engine_client_from_engine_args( (APIServer pid=34481) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (APIServer pid=34481) File "/opt/conda/lib/python3.12/contextlib.py", line 210, in __aenter__ (APIServer pid=34481) return await anext(self.gen) (APIServer pid=34481) ^^^^^^^^^^^^^^^^^^^^^ (APIServer pid=34481) File "/opt/conda/lib/python3.12/site-packages/vllm/entrypoints/openai/api_server.py", line 188, in build_async_engine_client_from_engine_args (APIServer pid=34481) async_llm = AsyncLLM.from_vllm_config( (APIServer pid=34481) ^^^^^^^^^^^^^^^^^^^^^^^^^^ (APIServer pid=34481) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/engine/async_llm.py", line 228, in from_vllm_config (APIServer pid=34481) return cls( (APIServer pid=34481) ^^^^ (APIServer pid=34481) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/engine/async_llm.py", line 155, in __init__ (APIServer pid=34481) self.engine_core = EngineCoreClient.make_async_mp_client( (APIServer pid=34481) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (APIServer pid=34481) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/engine/core_client.py", line 122, in make_async_mp_client (APIServer pid=34481) return AsyncMPClient(*client_args) (APIServer pid=34481) ^^^^^^^^^^^^^^^^^^^^^^^^^^^ (APIServer pid=34481) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/engine/core_client.py", line 819, in __init__ (APIServer pid=34481) super().__init__( (APIServer pid=34481) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/engine/core_client.py", line 479, in __init__ (APIServer pid=34481) with launch_core_engines(vllm_config, executor_class, log_stats) as ( (APIServer pid=34481) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ (APIServer pid=34481) File "/opt/conda/lib/python3.12/contextlib.py", line 144, in __exit__ (APIServer pid=34481) next(self.gen) (APIServer pid=34481) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 933, in launch_core_engines (APIServer pid=34481) wait_for_engine_startup( (APIServer pid=34481) File "/opt/conda/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 992, in wait_for_engine_startup (APIServer pid=34481) raise RuntimeError( (APIServer pid=34481) RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}