Delphi中高效遍历千万行CSV并提取指定列的最优方案

我有一个超过1000万行数据的CSV文件，需要提取其中第二列的数据，并将该列所有值保存为逗号分隔的文本文件。自己写的Delphi代码能实现需求，但运行速度极慢，单次执行耗时超过一天。

CSV文件示例 #

"1","facebook.com","10.00"
"2","fonts.googleapis.com","10.00"
"3","google.com","10.00"
"4","youtube.com","10.00"
"5","twitter.com","10.00"
"6","instagram.com","10.00"
"7","googletagmanager.com","10.00"
"8","linkedin.com","10.00"
"9","fonts.gstatic.com","10.00"
"10","gmpg.org","10.00"
"11","ajax.googleapis.com","9.47"
"12","maps.google.com","9.21"
"13","youtu.be","9.09"
"14","support.google.com","9.04"
"15","wordpress.org","9.01"
"16","cdnjs.cloudflare.com","9.01"
"17","play.google.com","9.00"
"18","plus.google.com","8.97"
"19","en.wikipedia.org","8.91"
"20","pinterest.com","8.81"
"21","docs.google.com","8.64"
"22","drive.google.com","8.64"
"23","s.w.org","8.59"
"24","bit.ly","8.53"
"25","github.com","8.50"
"26","goo.gl","8.49"
"27","secure.gravatar.com","8.45"
"28","amazon.com","8.39"
"29","itunes.apple.com","8.38"
"30","vimeo.com","8.38"
"31","developers.google.com","8.36"
"32","policies.google.com","8.31"
"33","vk.com","8.20"
"34","accounts.google.com","8.17"
"35","t.co","8.12"
"36","medium.com","8.10"
"37","flickr.com","8.09"
"38","creativecommons.org","8.07"
"39","w3.org","8.06"
"40","reddit.com","8.05"
"41","code.jquery.com","8.02"
"42","ec.europa.eu","7.98"
"43","gstatic.com","7.97"
"44","google-analytics.com","7.95"
"45","soundcloud.com","7.93"

期望输出示例 #

facebook.com,fonts.googleapis.com,google.com,youtube.com,twitter.com,instagram.com,googletagmanager.com,linkedin.com,fonts.gstatic.com,gmpg.org,ajax.googleapis.com,maps.google.com,youtu.be,support.google.com,wordpress.org,cdnjs.cloudflare.com,play.google.com,plus.google.com,en.wikipedia.org,pinterest.com,docs.google.com,drive.google.com,s.w.org,bit.ly,github.com,goo.gl,secure.gravatar.com,amazon.com,itunes.apple.com,vimeo.com,developers.google.com,policies.google.com,vk.com,accounts.google.com,t.co,medium.com,flickr.com,creativecommons.org,w3.org,reddit.com,code.jquery.com,ec.europa.eu,gstatic.com,google-analytics.com,soundcloud.com,

原Delphi代码（性能极差） #

begin
  var sites := 'D:\Downloads\domains.csv';
  var sitesList := TStringList.Create;
  sitesList.LoadFromFile(sites);


  var sitesNewList := TStringList.Create;
  for var I in sitesList do
  begin
    sitesNewList.Text := Trim(sitesNewList.Text) +
                         I.Substring(I.IndexOf(',')+2,(I.LastIndexOf(',')-I.IndexOf(','))-3) +
                         ',';
  end;

  sitesNewList.SaveToFile(TPath.ChangeExtension(sites,'txt'));
  sitesNewList.Free;

  sitesList.Free;
end;

请问实现该需求的最快方式是什么？如何快速遍历1000万+行数据并提取第二列？

原代码性能差的核心原因 #

1. 全量加载内存开销极大：TStringList.LoadFromFile把1000万行全部加载到内存，不仅占用大量内存，逐行解析过程本身效率就低。
2. 字符串拼接方式致命低效：每次循环都对sitesNewList.Text赋值，本质是反复创建整个字符串，1000万次拼接会产生指数级的内存拷贝和垃圾回收，这是性能崩溃的核心原因。
3. 索引计算冗余：重复计算字符串索引，截取逻辑也可以更高效。

最快实现方式：流处理+高效字符串定位 #

直接操作文件流，按块读取解析，定位第二列内容后直接写入输出流，避免全量加载内存，同时彻底抛弃低效的字符串拼接逻辑。

优化后的Delphi代码

uses
  System.SysUtils, System.IOUtils, System.Classes;

procedure ExtractSecondColumn(const InputFile, OutputFile: string);
var
  InputStream: TFileStream;
  OutputStream: TFileStream;
  Buffer: array[0..8191] of Char;
  BytesRead: Integer;
  LineBuffer: string;
  Pos1, Pos2: Integer;
  Domain: string;
begin
  InputStream := TFileStream.Create(InputFile, fmOpenRead or fmShareDenyWrite);
  OutputStream := TFileStream.Create(OutputFile, fmCreate or fmShareDenyWrite);
  try
    LineBuffer := '';
    while InputStream.Position < InputStream.Size do
    begin
      BytesRead := InputStream.Read(Buffer, SizeOf(Buffer));
      LineBuffer := LineBuffer + string(Buffer, 0, BytesRead);
      
      while True do
      begin
        Pos1 := Pos(#10, LineBuffer);
        if Pos1 = 0 then Break;

        // 提取并处理当前行
        var CurrentLine := Copy(LineBuffer, 1, Pos1 - 1);
        LineBuffer := Copy(LineBuffer, Pos1 + 1, MaxInt);
        CurrentLine := TrimRight(CurrentLine);
        
        if CurrentLine.IsEmpty then Continue;

        // 定位第二列：跳过第一个逗号，找到第二个双引号开始，到下一个",结束
        Pos1 := Pos(',', CurrentLine);
        if Pos1 = 0 then Continue;
        Pos1 := Pos('"', CurrentLine, Pos1 + 1);
        if Pos1 = 0 then Continue;
        Pos2 := Pos('",', CurrentLine, Pos1 + 1);
        if Pos2 = 0 then Continue;

        Domain := Copy(CurrentLine, Pos1 + 1, Pos2 - Pos1 - 1);
        // 直接写入输出流，避免中间字符串拼接
        OutputStream.WriteBuffer(PChar(Domain)^, Length(Domain) * SizeOf(Char));
        OutputStream.WriteBuffer(',', SizeOf(Char));
      end;
    end;

    // 处理文件末尾剩余的未换行内容
    if not LineBuffer.IsEmpty then
    begin
      var CurrentLine := TrimRight(LineBuffer);
      if not CurrentLine.IsEmpty then
      begin
        Pos1 := Pos(',', CurrentLine);
        if Pos1 > 0 then
        begin
          Pos1 := Pos('"', CurrentLine, Pos1 + 1);
          if Pos1 > 0 then
          begin
            Pos2 := Pos('",', CurrentLine, Pos1 + 1);
            if Pos2 > 0 then
            begin
              Domain := Copy(CurrentLine, Pos1 + 1, Pos2 - Pos1 - 1);
              OutputStream.WriteBuffer(PChar(Domain)^, Length(Domain) * SizeOf(Char));
              OutputStream.WriteBuffer(',', SizeOf(Char));
            end;
          end;
        end;
      end;
    end;
  finally
    OutputStream.Free;
    InputStream.Free;
  end;
end;

// 调用示例
begin
  ExtractSecondColumn('D:\Downloads\domains.csv', 'D:\Downloads\domains.txt');
end;

进一步优化方向 #

1. 调整缓冲区大小：可以把Buffer的大小调整为16384或32768，减少磁盘IO的次数。
2. 适配ASCII编码：如果CSV是ASCII编码，改用AnsiChar和TAnsiString可以进一步降低内存占用和拷贝开销。
3. 多线程并行解析：如果磁盘IO不是瓶颈，可以将文件分割为多个块，用多线程并行解析，注意保证输出顺序的一致性。
4. 简化行处理逻辑：如果确定行尾只有#10或#13，可以去掉TrimRight操作，直接截取内容。

方案高效的核心原因 #

• 内存占用极低：不需要加载整个文件到内存，仅保留当前处理的行缓冲区。
• 底层流操作：读写都是直接调用系统底层流接口，避免TStringList的额外封装开销。
• 精准字符串定位：用Pos函数直接定位分隔符，避免冗余的索引计算和字符串截取操作。
• 无低效拼接：提取的内容直接写入输出流，彻底避免了中间字符串的反复创建与拷贝。

内容的提问来源于stack exchange，提问作者Shaun Roselt