Python如何将文件所有行存入dataset_texts与dataset_labels列表

我尝试从输入文件中读取所有行，分别存入列表dataset_texts和dataset_labels。其中text_str获取行中的文本序列，labels_str保存对应文本序列的标签向量，label保存向量中1的位置，但目前仅能保存文件的最后一行。请告知如何让两个列表存储所有行及其对应标签向量中1的位置？

以下是逐行检查过的代码：

from transformers import BertTokenizer
import torch
import re

training_set_path = '../test.txt'

regexp = r'^(.*)\t(\d+)$'

dataset_texts = list()
dataset_labels = list()

input_file = open(training_set_path, 'rb' )
print("Dataset loaded")

num_labels = 0 
print("Num_labels")
print(num_labels)
#labels_str = []   # added by me 
for line in input_file:
    line = line.decode( errors = 'replace' )
    #print(line)
    if re.match(regexp, line):
      text_str = re.findall( regexp, line )[0][0]  # getting the aa sequence
      print("here text_str")
      print(text_str)
      labels_str = re.findall( regexp, line )[0][1] # getting the corresponding vector
      print("here labels_str")
      print(labels_str)
      label = labels_str.index('1')
      print("here label")
      print(label)
      dataset_texts.append( text_str )
      dataset_labels.append( label )
      num_labels = len(labels_str)
      print("Here length num_labels")
      print(num_labels)
      counter += 1

    # else:
    #   break
input_file.close()
print("______________________________________________________________________")
print("Here dataset_text")
print(dataset_texts)
print("Here dataset_labels")
print(dataset_labels)
output_file = open( logs_path, 'w')
num_labels = len(labels_str)

当前输出如下：

Dataset loaded
Num_labels
0
here text_str
Q Q L R K P A E E L G R E I T H Q L F L L G C G A Q M L K Y A S P P M A Q A W C Q V M L D T R G G V R L S E Q I Q N D L L
here labels_str
1000000000000000000000000000000000000000000000000000000000000
here label
0
Here length num_labels
61
______________________________________________________________________
Here dataset_text
['Q Q L R K P A E E L G R E I T H Q L F L L G C G A Q M L K Y A S P P M A Q A W C Q V M L D T R G G V R L S E Q I Q N D L L']
Here dataset_labels
[0]

当前代码只处理一行的核心原因是正则匹配受行尾空白/换行符干扰，加上存在未初始化变量的隐患。以下是修正后的代码及关键说明：

from transformers import BertTokenizer
import torch
import re

training_set_path = '../test.txt'
# 匹配前先清理行首尾空白，避免换行符、空格干扰正则匹配
regexp = r'^(.*)\t(\d+)$'

dataset_texts = list()
dataset_labels = list()

# 用with语句自动管理文件，无需手动关闭
with open(training_set_path, 'rb') as input_file:
    print("Dataset loaded")
    num_labels = 0 
    print("Num_labels")
    print(num_labels)
    counter = 0  # 初始化计数器，避免未定义报错
    for line in input_file:
        # 解码后去除行首尾所有空白（包括换行符、空格）
        line_clean = line.decode(errors='replace').strip()
        if not line_clean:  # 跳过空行
            continue
        # 用match对象直接提取内容，比两次findall更高效
        match_result = re.match(regexp, line_clean)
        if match_result:
            text_str = match_result.group(1)
            print("here text_str")
            print(text_str)
            labels_str = match_result.group(2)
            print("here labels_str")
            print(labels_str)
            # 捕获标签中无'1'的异常，避免程序崩溃
            try:
                label = labels_str.index('1')
                print("here label")
                print(label)
                dataset_texts.append(text_str)
                dataset_labels.append(label)
                num_labels = len(labels_str)
                print("Here length num_labels")
                print(num_labels)
                counter += 1
            except ValueError:
                print(f"警告：标签向量中未找到'1'：{labels_str}")

print("______________________________________________________________________")
print("Here dataset_text")
print(dataset_texts)
print("Here dataset_labels")
print(dataset_labels)
# 注意：logs_path需要提前定义，否则写入会报错，示例如下
# logs_path = '../process_log.txt'
# with open(logs_path, 'w') as output_file:
#     output_file.write(f"标签长度：{num_labels}\n")
#     output_file.write(f"处理样本数：{counter}\n")

关键修改点： #

1. 清理行内容：解码后用strip()去除行首尾空白（包括换行符），解决正则匹配失效问题
2. 优化正则提取：用re.match的结果直接调用group()获取内容，避免重复调用findall
3. 初始化变量：添加counter = 0，解决未定义变量报错
4. 异常处理：用try-except捕获index('1')可能抛出的异常，避免程序崩溃
5. 文件安全操作：用with语句管理文件，无需手动关闭，避免资源泄漏
6. 跳过空行：增加空行判断，避免处理无效内容

内容的提问来源于stack exchange，提问作者Vykov