使用Blosc压缩的HDF5文件读取时遇OSError问题求助
2026-6-12
Blosc2压缩HDF5文件读取报错：
OSError: Can't synchronously read data (can't open directory) 我用Python代码生成了采用Blosc2压缩的HDF5文件，生成过程无报错，但读取trainval组下的image_log数据集时触发上述错误。其他压缩方式可正常读取，已尝试重装h5py与hdf5plugin、执行磁盘检查、多次重新运行脚本，问题仍未解决，存储文件的D盘尚有300GB可用空间。
生成文件代码
# Create forecast training data file import h5py import hdf5plugin #mmap_array = np.memmap('video_prediction_224.dat', dtype='uint8', mode='w+', shape=(n_images, 224, 224, 3)) sampling_interval = 2 n_valid = 307518 # index 307517 is the last valid index n_trainval = 149680 n_test = 6145 chunk_shape = (1, stack_height+1, *output_img_shape) batch_size = 8000 #for sampling_interval in sampling_interval_all: resume_idx = 0 with h5py.File('video_prediction_224_testing.h5', 'w') as f: #image_log = f.create_dataset('image_log', shape = (n_images, 224, 224, 3), dtype ='uint8', ) if resume_idx and 'image_log' in f: # Resume mode - datasets already exist image_log_ds = f['image_log'] pv_log_ds = f['pv_log'] pv_pred_ds = f['pv_pred'] print(f"Resuming from existing datasets. Current size: {image_log_ds.shape[0]}") else: # First run - create new datasets trainval_group = f.create_group('trainval') test_group = f.create_group('test') image_log_trainval_ds = trainval_group.create_dataset( 'image_log', shape=(n_trainval, stack_height+1, *output_img_shape), chunks=chunk_shape, compression=hdf5plugin.Blosc2(cname='zstd', clevel=1, filters=hdf5plugin.Blosc2.SHUFFLE), dtype='uint8' ) pv_log_trainval_ds = trainval_group.create_dataset( 'pv_log', shape=(n_trainval, stack_height+1), dtype='float64' ) pv_pred_trainval_ds = trainval_group.create_dataset( 'pv_pred', shape=(n_trainval,), dtype='float64' ) image_log_test_ds = test_group.create_dataset( 'image_log', shape=(n_test, stack_height+1, *output_img_shape), chunks=chunk_shape, compression=hdf5plugin.Blosc2(cname='zstd', clevel=1, filters=hdf5plugin.Blosc2.SHUFFLE), dtype='uint8' ) pv_log_test_ds = test_group.create_dataset( 'pv_log', shape=(n_test, stack_height+1), dtype='float64' ) pv_pred_test_ds = test_group.create_dataset( 'pv_pred', shape=(n_test,), dtype='float64' ) print("Creating new datasets") #image_log = np.empty([0,stack_height+1]+output_img_shape,dtype = 'uint8') #pv_log = np.empty([0, stack_height+1]) #pv_pred = np.empty([0]) last_valid_index = 0 curr_trainval_size = 0 curr_test_size = 0 tic = time.process_time() for b in range(resume_idx if (resume_idx and 'image_log' in f) else 0, 8000, batch_size): current_batch_size = min(batch_size, n_images-b) # Initialize variables to save pv values #image_log_batch = np.zeros([n_images,stack_height+1]+output_img_shape,dtype = 'uint8') idx_test_batch = get_subarray_between_values(np.asarray(idx_test),b,b+current_batch_size) image_log_batch = np.zeros([current_batch_size,stack_height+1]+output_img_shape,dtype = 'uint8') #all_times_batch = all_times[b*batch_size : b*batch_size + current_batch_size] pv_log_batch = np.zeros((current_batch_size,stack_height+1)) pv_pred_batch = np.zeros(current_batch_size) validity_mask = np.ones(current_batch_size,dtype = bool) sampling_interval_td = datetime.timedelta(minutes = sampling_interval) - datetime.timedelta(seconds=1) for i in range(current_batch_size): count = b+i # See if the specified sampling frequency is met if all_times[count] - all_times[last_valid_index] > sampling_interval_td: # Collecting groud truth for predicted value pred_time = all_times[count]+datetime.timedelta(minutes=forecast_horizon) pv_pred_idx = find_time_within_nparray(pv_data.index,pred_time) if pv_pred_idx is None:# if prediction ground truth not found validity_mask[i] = False #print(all_times[i],'has no PV pred') else: pv_pred_batch[i] = pv_data.iloc[pv_pred_idx] # Collecting image log and PV log for j in range(stack_height+1): log_time = all_times[count] - datetime.timedelta(minutes = j) # Collecting a stack of image log_time_idx = find_time_within_nparray(all_times,log_time) if log_time_idx is not None: image_log_batch[i,j] = all_images[log_time_idx] else: validity_mask[i] = False #print(all_times[count],'has no image log') break # Collecting a stack of PV value pv_log_idx = find_time_within_nparray(pv_data.index,log_time) # Check if PV value present if pv_log_idx is None: validity_mask[i] = False #print(all_times[count],'has no PV log') break else: pv_log_batch[i,j] = pv_data.iloc[pv_log_idx] else: # if this is in between the sampling points, discard validity_mask[i] = False if validity_mask[i]: last_valid_index = count # Prompt progress of current work print('processed {0}/{1} images'.format(b+current_batch_size,len(all_times))) # Only pick out the valid time points #all_times_batch = all_times_batch[validity_mask] test_mask = np.zeros(current_batch_size, dtype=bool) test_mask[idx_test_batch-b] = True validity_test_mask = validity_mask & test_mask trainval_mask = np.ones(current_batch_size, dtype=bool) trainval_mask[idx_test_batch-b] = False validity_trainval_mask = validity_mask & trainval_mask image_log_trainval_batch = image_log_batch[validity_trainval_mask] pv_log_trainval_batch = pv_log_batch[validity_trainval_mask] pv_pred_trainval_batch = pv_pred_batch[validity_trainval_mask] image_log_test_batch = image_log_batch[validity_test_mask] pv_log_test_batch = pv_log_batch[validity_test_mask] pv_pred_test_batch = pv_pred_batch[validity_test_mask] # Store information print("storing data") #image_log_ds[curr_size:curr_size+validity_mask.sum()] = image_log_batch #pv_log_ds[curr_size:curr_size+validity_mask.sum()] = pv_log_batch #pv_pred_ds[curr_size:curr_size+validity_mask.sum()] = pv_pred_batch image_log_trainval_ds[curr_trainval_size:curr_trainval_size+validity_trainval_mask.sum()] = image_log_trainval_batch #print("image_log_trainval_batch: ",image_log_trainval_batch) #print("image_log_trainval_ds.shape: ",image_log_trainval_ds.shape) print("stored trainval image log") pv_log_trainval_ds[curr_trainval_size:curr_trainval_size+validity_trainval_mask.sum()] = pv_log_trainval_batch print("stored trainval pv log") pv_pred_trainval_ds[curr_trainval_size:curr_trainval_size+validity_trainval_mask.sum()] = pv_pred_trainval_batch print("stored trainval pv pred") image_log_test_ds[curr_test_size:curr_test_size+validity_test_mask.sum()] = image_log_test_batch print("stored test image log") pv_log_test_ds[curr_test_size:curr_test_size+validity_test_mask.sum()] = pv_log_test_batch pv_pred_test_ds[curr_test_size:curr_test_size+validity_test_mask.sum()] = pv_pred_test_batch curr_trainval_size += validity_trainval_mask.sum() print("trainval_size: ",validity_trainval_mask.sum()) curr_test_size += validity_test_mask.sum() print("test_size: ",validity_test_mask.sum()) print('For sampling frequency: ',sampling_interval,' minutes') #print('Expected finishing time:', datetime.datetime.now()+ # datetime.timedelta(seconds = (time.process_time() - tic)*(len(all_times)/(b+batch_size)))) f.flush() del image_log_trainval_batch del pv_log_trainval_batch del pv_pred_trainval_batch del image_log_test_batch del pv_log_test_batch del pv_pred_test_batch pred_folder_child = os.path.join(pred_folder,'frequency_'+str(sampling_interval)) #store_trainval_test(all_times,image_log,pv_log,pv_pred,pred_folder_child)
读取代码
# generate handler for the hdf5 data forecast_dataset = h5py.File(data_path, 'r') # show structure of the hdf5 data def get_all(name): if name!=None: print(forecast_dataset[name]) forecast_dataset.visit(get_all) img = forecast_dataset["trainval"]["image_log"][0] print(img)
报错信息
data_folder: d:\PVOutputPrediction\data data_path: d:\PVOutputPrediction\video_prediction_224_second.h5 output_folder: d:\PVOutputPrediction\model_output\UNet_sky_image_PV_mapping <HDF5 group "/test" (3 members)> <HDF5 dataset "image_log": shape (6145, 16, 224, 224, 3), type "|u1"> <HDF5 dataset "pv_log": shape (6145, 16), type "<f8"> <HDF5 dataset "pv_pred": shape (6145,), type "<f8"> <HDF5 group "/trainval" (3 members)> <HDF5 dataset "image_log": shape (149680, 16, 224, 224, 3), type "|u1"> <HDF5 dataset "pv_log": shape (149680, 16), type "<f8"> <HDF5 dataset "pv_pred": shape (149680,), type "<f8"> --------------------------------------------------------------------------- OSError Traceback (most recent call last) Cell In[7], line 3 1 import matplotlib.pyplot as plt ----> 3 img = forecast_dataset["trainval"]["image_log"][0] 4 print(img) File h5py/_objects.pyx:56, in h5py._objects.with_phil.wrapper() File h5py/_objects.pyx:57, in h5py._objects.with_phil.wrapper() File d:\PVOutputPrediction\pytorch_env\Lib\site-packages\h5py\_hl\dataset.py:820, in Dataset.__getitem__(self, args, new_dtype) 818 if self._fast_read_ok and (new_dtype is None): 819 try: --> 820 return self._fast_reader.read(args) 821 except TypeError: 822 pass # Fall back to Python read pathway below File h5py/_selector.pyx:376, in h5py._selector.Reader.read() OSError: Can't synchronously read data (can't open directory)
解决方案
调整Blosc2压缩参数
尝试更换压缩算法或关闭SHUFFLE过滤器，避免潜在的索引兼容问题：# 更换为lz4压缩算法 compression=hdf5plugin.Blosc2(cname='lz4', clevel=1) # 或关闭SHUFFLE过滤器 compression=hdf5plugin.Blosc2(cname='zstd', clevel=1, filters=[])修改分块大小
当前单条数据作为一个分块的设置可能导致读取时的目录索引错误，尝试使用更合理的分块尺寸，比如：chunk_shape = (32, 16, 224, 224, 3) # 根据stack_height调整第二个维度规避h5py快速读取
报错来自h5py的快速读取模块，可通过强制使用Python读取路径绕过：# 用切片再取索引的方式触发Python读取路径 img = forecast_dataset["trainval"]["image_log"][0:1][0] # 或直接转换为numpy数组 img = np.array(forecast_dataset["trainval"]["image_log"])[0]更新依赖库版本
旧版本hdf5plugin可能存在Blosc2兼容性问题，执行升级：pip install --upgrade hdf5plugin h5py校验文件完整性
使用h5py内置的校验功能检查文件是否损坏：import h5py with h5py.File(data_path, 'r') as f: f.verify()若校验失败，说明文件生成过程存在隐性中断，需重新生成并确保写入过程无异常。
内容的提问来源于stack exchange，提问作者Aiden Yun
相关产品推荐
相关产品推荐
