Dask LocalCUDACluster启动失败：AttributeError问题求助

安全阿岳

2026-7-31

Dask LocalCUDACluster启动报错：AttributeError: 'str' object has no attribute 'decode'

运行以下代码时触发错误：

from dask.distributed import Client
from dask_cuda import LocalCUDACluster
cluster = LocalCUDACluster()
client = Client(cluster)

错误栈信息：

AttributeError                            Traceback (most recent call last)
File ~/miniconda3/envs/rapids-23.04/lib/python3.10/site-packages/distributed/deploy/spec.py:319, in SpecCluster._start(self)
    318     cls = import_term(cls)
---> 319 self.scheduler = cls(**self.scheduler_spec.get("options", {}))
    320 self.scheduler = await self.scheduler

File ~/miniconda3/envs/rapids-23.04/lib/python3.10/site-packages/distributed/scheduler.py:3481, in Scheduler.__init__(self, loop, delete_interval, synchronize_worker_interval, services, service_kwargs, allowed_failures, extensions, validate, scheduler_file, security, worker_ttl, idle_timeout, interface, host, port, protocol, dashboard_address, dashboard, http_prefix, preload, preload_argv, plugins, contact_address, transition_counter_max, jupyter, **kwargs)
   3480 if show_dashboard:
-> 3481     distributed.dashboard.scheduler.connect(
   3482         self.http_application, self.http_server, self, prefix=http_prefix
   3483     )
   3484 self.jupyter = jupyter

File ~/miniconda3/envs/rapids-23.04/lib/python3.10/site-packages/distributed/dashboard/scheduler.py:158, in connect(application, http_server, scheduler, prefix)
    156 def connect(application, http_server, scheduler, prefix=""):
    157     bokeh_app = BokehApplication(
-> 158         applications, scheduler, prefix=prefix, template_variables=template_variables()
    159     )
    160     application.add_application(bokeh_app)

File ~/miniconda3/envs/rapids-23.04/lib/python3.10/site-packages/cytoolz/functoolz.pyx:475, in cytoolz.functoolz._memoize.__call__()

File ~/miniconda3/envs/rapids-23.04/lib/python3.10/site-packages/distributed/dashboard/scheduler.py:131, in template_variables()
    123 from distributed.diagnostics.nvml import device_get_count
    125 template_variables = {
    126     "pages": [
    127         "status",
    128         "workers",
    129         "tasks",
    130         "system",
-> 131         *(["gpu"] if device_get_count() > 0 else []),
    132         "profile",
    133         "graph",
    134         "groups",
    135         "info",
    136     ],
    137     "plots": [
    138         {
    139             "url": x.strip("/"),
    140             "name": " ".join(x.strip("/").split("-")[1:])
    141             .title()
    142             .replace("Cpu", "CPU")
    143             .replace("Gpu", "GPU"),
    144         }
    145         for x in applications
    146         if "individual" in x
    147     ]
    148     + [{"url": "hardware", "name": "Hardware"}],
    149 }
    150 template_variables["plots"] = sorted(
    151     template_variables["plots"], key=lambda d: d["name"]
    152 )

File ~/miniconda3/envs/rapids-23.04/lib/python3.10/site-packages/distributed/diagnostics/nvml.py:126, in device_get_count()
    125 def device_get_count():
-> 126     init_once()
    127     if not is_initialized():

File ~/miniconda3/envs/rapids-23.04/lib/python3.10/site-packages/distributed/diagnostics/nvml.py:108, in init_once()
    105     return
    107 if _in_wsl() and parse_version(
-> 108     pynvml.nvmlSystemGetDriverVersion().decode()
    109 ) < parse_version(MINIMUM_WSL_VERSION):
    110     NVML_STATE = NVMLState.DISABLED_WSL_INSUFFICIENT_DRIVER

AttributeError: 'str' object has no attribute 'decode'

上述异常是触发后续异常的直接原因：

RuntimeError                              Traceback (most recent call last)
Cell In[22], line 3
      1 from dask_cuda import LocalCUDACluster
----> 3 cluster = LocalCUDACluster()
      4 client = Client(cluster)

File ~/miniconda3/envs/rapids-23.04/lib/python3.10/site-packages/dask_cuda/local_cuda_cluster.py:336, in LocalCUDACluster.__init__(self, CUDA_VISIBLE_DEVICES, n_workers, threads_per_worker, memory_limit, device_memory_limit, data, local_directory, shared_filesystem, protocol, enable_tcp_over_ucx, enable_infiniband, enable_nvlink, enable_rdmacm, rmm_pool_size, rmm_maximum_pool_size, rmm_managed_memory, rmm_async, rmm_log_directory, rmm_track_allocations, jit_unspill, log_spilling, worker_class, pre_import, **kwargs)
    329     worker_class = partial(
    330         LoggedNanny if log_spilling is True else Nanny,
    331         worker_class=worker_class,
    332     )
    334 self.pre_import = pre_import
-> 336 super().__init__(
    337     n_workers=0,
    338     threads_per_worker=threads_per_worker,
    339     memory_limit=self.memory_limit,
    340     processes=True,
    341     data=data,
    342     local_directory=local_directory,
    343     protocol=protocol,
    344     worker_class=worker_class,
    345     config={
    346         "distributed.comm.ucx": get_ucx_config(
    347             enable_tcp_over_ucx=enable_tcp_over_ucx,
    348             enable_nvlink=enable_nvlink,
    349             enable_infiniband=enable_infiniband,
    350             enable_rdmacm=enable_rdmacm,
    351         )
    352     },
    353     **kwargs,
    354 )
    356 self.new_spec["options"]["preload"] = self.new_spec["options"].get(
    357     "preload", []
    358 ) + ["dask_cuda.initialize"]
    359 self.new_spec["options"]["preload_argv"] = self.new_spec["options"].get(
    360     "preload_argv", []
    361 ) + ["--create-cuda-context"]

File ~/miniconda3/envs/rapids-23.04/lib/python3.10/site-packages/distributed/deploy/local.py:253, in LocalCluster.__init__(self, name, n_workers, threads_per_worker, processes, loop, start, host, ip, scheduler_port, silence_logs, dashboard_address, worker_dashboard_address, diagnostics_port, services, worker_services, service_kwargs, asynchronous, security, protocol, blocked_handlers, interface, worker_class, scheduler_kwargs, scheduler_sync_interval, **worker_kwargs)
    250 worker = {"cls": worker_class, "options": worker_kwargs}
    251 workers = {i: worker for i in range(n_workers)}
-> 253 super().__init__(
    254     name=name,
    255     scheduler=scheduler,
    256     workers=workers,
    257     worker=worker,
    258     loop=loop,
    259     asynchronous=asynchronous,
    260     silence_logs=silence_logs,
    261     security=security,
    262     scheduler_sync_interval=scheduler_sync_interval,
    263 )

File ~/miniconda3/envs/rapids-23.04/lib/python3.10/site-packages/distributed/deploy/spec.py:286, in SpecCluster.__init__(self, workers, scheduler, worker, asynchronous, loop, security, silence_logs, name, shutdown_on_close, scheduler_sync_interval)
    284 if not called_from_running_loop:
    285     self._loop_runner.start()
-> 286     self.sync(self._start)
    287     try:
    288         self.sync(self._correct_state)

File ~/miniconda3/envs/rapids-23.04/lib/python3.10/site-packages/distributed/utils.py:338, in SyncMethodMixin.sync(self, func, asynchronous, callback_timeout, *args, **kwargs)
    336     return future
    337 else:
-> 338     return sync(
    339         self.loop, func, *args, callback_timeout=callback_timeout, **kwargs
    340     )

File ~/miniconda3/envs/rapids-23.04/lib/python3.10/site-packages/distributed/utils.py:405, in sync(loop, func, callback_timeout, *args, **kwargs)
    403 if error:
    404     typ, exc, tb = error
-> 405     raise exc.with_traceback(tb)
    406 else:
    407     return result

File ~/miniconda3/envs/rapids-23.04/lib/python3.10/site-packages/distributed/utils.py:378, in sync.<locals>.f()
    376         future = asyncio.wait_for(future, callback_timeout)
    377     future = asyncio.ensure_future(future)
-> 378     result = yield future
    379 except Exception:
    380     error = sys.exc_info()

File ~/miniconda3/envs/rapids-23.04/lib/python3.10/site-packages/tornado/gen.py:769, in Runner.run(self)
    766 exc_info = None
    768 try:
-> 769     value = future.result()
    770 except Exception:
    771     exc_info = sys.exc_info()

File ~/miniconda3/envs/rapids-23.04/lib/python3.10/site-packages/distributed/deploy/spec.py:330, in SpecCluster._start(self)
    328 self.status = Status.failed
    329 await self._close()
-> 330 raise RuntimeError(f"Cluster failed to start: {e}") from e

RuntimeError: 集群启动失败：'str'对象没有属性'decode'

环境信息：

Dask版本：2023.2.0
已尝试的解决方法：
- 重装rapidsai
- 将Python版本从3.10降级至3.8
- 调整LocalCUDACluster()的参数

以上方法均未解决问题，寻求技术支持。

内容的提问来源于stack exchange，提问作者LRyougiShikiZ