如何基于标记字符串使用Pandas从文本文件提取目标CSV数据？

针对你需要处理数百个标记行位置不固定的文本文件，这里提供几个实用的解决思路：

方法1：先定位标记行号，再用read_csv读取 #

先遍历文件找到目标数据的起始和结束行号，再传入read_csv的skiprows和nrows参数。这种方法适合需要保留read_csv完整参数配置（比如指定分隔符、列名）的场景：

import pandas as pd

def get_target_row_range(file_path, start_marker="BREAK THROUGH @ WT, ITERATION", end_marker="END BREAK THROUGH @ WT"):
    start_row = None
    end_row = None
    with open(file_path, 'r') as f:
        for idx, line in enumerate(f):
            stripped_line = line.strip()
            if start_marker in stripped_line:
                start_row = idx + 1  # 跳过标记行本身，从下一行开始读取数据
            if end_marker in stripped_line:
                end_row = idx
                break  # 找到结束标记后立即停止遍历，提升效率
    return start_row, end_row

# 处理单个文件
file_path = "model_output.txt"
start, end = get_target_row_range(file_path)
if start and end:
    df = pd.read_csv(
        file_path,
        skiprows=start,
        nrows=end - start,
        # 可添加其他read_csv参数，比如sep=',', header=0等
    )
else:
    print(f"文件 {file_path} 未找到目标标记行")

方法2：直接捕获目标内容，内存中转换为CSV #

跳过行号定位，直接在遍历文件时捕获两个标记之间的内容，再用StringIO包装后传给read_csv。这种方法只需遍历文件一次，效率更高：

from io import StringIO
import pandas as pd

def extract_target_data(file_path, start_marker="BREAK THROUGH @ WT, ITERATION", end_marker="END BREAK THROUGH @ WT"):
    target_lines = []
    capture_mode = False
    with open(file_path, 'r') as f:
        for line in f:
            stripped_line = line.strip()
            if start_marker in stripped_line:
                capture_mode = True
                continue  # 跳过标记行，不加入目标内容
            if end_marker in stripped_line:
                capture_mode = False
                break
            if capture_mode:
                target_lines.append(line)
    # 将捕获的内容转为CSV格式的DataFrame
    if target_lines:
        return pd.read_csv(StringIO(''.join(target_lines)))
    else:
        return None

# 调用示例
df = extract_target_data("model_output.txt")
if df is not None:
    print(df.head())
else:
    print("未提取到有效数据")

方法3：批量处理数百个文件 #

结合glob遍历所有目标文件，循环调用上述提取函数，最后合并所有数据：

import glob
from io import StringIO
import pandas as pd

def process_single_file(file_path):
    target_lines = []
    capture_mode = False
    start_marker = "BREAK THROUGH @ WT, ITERATION"
    end_marker = "END BREAK THROUGH @ WT"
    with open(file_path, 'r') as f:
        for line in f:
            stripped_line = line.strip()
            if start_marker in stripped_line:
                capture_mode = True
                continue
            if end_marker in stripped_line:
                capture_mode = False
                break
            if capture_mode:
                target_lines.append(line)
    if target_lines:
        df = pd.read_csv(StringIO(''.join(target_lines)))
        df['source_file'] = file_path  # 添加源文件列，方便后续溯源
        return df
    else:
        print(f"跳过无效文件：{file_path}")
        return None

# 遍历指定目录下的所有txt文件
all_files = glob.glob("/path/to/your/model/files/*.txt")
all_dataframes = []

for file in all_files:
    df = process_single_file(file)
    if df is not None:
        all_dataframes.append(df)

# 合并所有数据并保存
if all_dataframes:
    combined_df = pd.concat(all_dataframes, ignore_index=True)
    combined_df.to_csv("combined_model_results.csv", index=False)
    print(f"已合并 {len(all_dataframes)} 个文件的数据，保存至combined_model_results.csv")
else:
    print("未找到任何有效数据文件")

注意事项 #

• 确保标记行的匹配逻辑准确：如果标记行可能包含多余空格或大小写差异，可以调整判断条件（比如stripped_line == start_marker精确匹配，或者忽略大小写）。
• 处理大文件时，逐行读取的方式比一次性加载整个文件更节省内存。

内容的提问来源于stack exchange，提问作者Rohit Warrier