Azure Blob文件迁移后数据损坏问题排查求助

我在Azure Blob中存储了一些.log.gz格式的文件，处理完成后将其迁移至另一个容器，但目标容器中的文件显示为乱码。请问我哪里操作有误？

源文件为.log.gz格式。

初始代码 #

[Function("Function1")]
 public async Task Run([TimerTrigger("0 */5 * * * *", RunOnStartup = true)] TimerInfo myTimer)
 {
     //Reading the config file. For local it is local.settings.json
     var config = new ConfigurationBuilder()
         .AddJsonFile("local.settings.json", optional: true, reloadOnChange: true)
         .AddEnvironmentVariables()
         .Build();

     string blbobConnectionString = config["AzureWebJobsStorage"];
     BlobContainerClient blobContainerClient = new BlobContainerClient(blbobConnectionString, "abc");
     BlobContainerClient destinationBlobContainerClient = new BlobContainerClient(blbobConnectionString, "xyz");


     var blobs = blobContainerClient.GetBlobsAsync();
     await foreach (var blob in blobs)
     {
         _logger.LogInformation($"Blob Name: {blob.Name}");
         BlobClient blobClient = blobContainerClient.GetBlobClient(blob.Name);

         using (Stream blobStream = await blobClient.OpenReadAsync())
         using (GZipStream decompressionStream = new GZipStream(blobStream, CompressionMode.Decompress))
         using (StreamReader reader = new StreamReader(decompressionStream, Encoding.Default))
         {
             string line;
             while ((line = await reader.ReadLineAsync()) != null)
             {
                 // Process the line here
                 Console.WriteLine(line);
             }

             BlobClient destinationBlobClient = destinationBlobContainerClient.GetBlobClient(blob.Name);
             blobStream.Position = 0;
             await destinationBlobClient.UploadAsync(blobStream);
         }
         blobClient.DeleteAsync(DeleteSnapshotsOption.None);
     }
 }

编辑1：重新读取Blob上传但问题依旧 #

更新后的代码如下：

[Function("Function1")]
  public async Task Run([TimerTrigger("0 */5 * * * *", RunOnStartup = true)] TimerInfo myTimer)
  {
      _logger.LogInformation($"C# Timer trigger function executed at: {DateTime.Now}");
      
      if (myTimer.ScheduleStatus is not null)
      {
          _logger.LogInformation($"Next timer schedule at: {myTimer.ScheduleStatus.Next}");
      }

      //Reading the config file. For local it is local.settings.json
      var config = new ConfigurationBuilder()
          .AddJsonFile("local.settings.json", optional: true, reloadOnChange: true)
          .AddEnvironmentVariables()
          .Build();

      string blbobConnectionString = config["AzureWebJobsStorage"];

      BlobContainerClient blobContainerClient = new BlobContainerClient(blbobConnectionString, "abc");

      BlobContainerClient destinationBlobContainerClient = new BlobContainerClient(blbobConnectionString, "xyz");


      var blobs = blobContainerClient.GetBlobsAsync();
      await foreach (var blob in blobs)
      {
          _logger.LogInformation($"Blob Name: {blob.Name}");
          BlobClient blobClient = blobContainerClient.GetBlobClient(blob.Name);

          using (Stream blobStream = await blobClient.OpenReadAsync())
          using (GZipStream decompressionStream = new GZipStream(blobStream, CompressionMode.Decompress))
          using (StreamReader reader = new StreamReader(decompressionStream, Encoding.Default))
          {
              string line;
              while ((line = await reader.ReadLineAsync()) != null)
              {
                  // Process the line here
                  Console.WriteLine(line);
                  JObject json = JObject.Parse(line);
                  string dateOfEvent = System.Convert.ToString( json["Datetime"]);
                  string datePattern = "M/d/yyyy h:m:s tt";
                  DateTime parsedDate;

                  if (DateTime.TryParseExact(dateOfEvent, datePattern, null,  DateTimeStyles.None, out parsedDate))
                      Console.WriteLine("Parsed date: "+ parsedDate);

                  json["Datetime"] = parsedDate;

                  line = json.ToString();

                  Console.WriteLine(System.DateTime.UtcNow.ToString("YYYY-MM-DDThh:mm:ssZ"));
                 dateOfEvent = parsedDate.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture);

                  var datestring = DateTime.UtcNow.ToString("r");
                  var jsonBytes = Encoding.UTF8.GetBytes(line);
                  string stringToHash = "POST\n" + jsonBytes.Length + "\napplication/json\n" + "x-ms-date:" + datestring + "\n/api/logs";
                  string hashedString = BuildSignature(stringToHash, sharedKey);
                  string signature = "SharedKey " + customerId + ":" + hashedString;
          
                  PostData(signature, datestring, line);
              }
          }


          BlobClient destinationBlobClient = destinationBlobContainerClient.GetBlobClient(blob.Name);
          //blobStream.Position = 0;
          await destinationBlobClient.UploadAsync(await blobClient.OpenReadAsync());

          blobClient.DeleteAsync(DeleteSnapshotsOption.None);
      }
  }

核心问题总结 #

乱码的根源在于流操作逻辑错误或编码/文件类型设置不当，分两种场景解决：

场景1：仅迁移原始压缩文件 #

如果不需要修改日志内容，直接复制原始Blob是最高效且无错误的方式，避免流操作带来的问题：

// 替换原上传逻辑，使用Blob原生复制方法
BlobClient destinationBlobClient = destinationBlobContainerClient.GetBlobClient(blob.Name);
await destinationBlobClient.StartCopyFromUriAsync(blobClient.Uri);

// 可选：等待复制完成再删除原文件
await WaitForCopyCompletion(destinationBlobClient);
await blobClient.DeleteAsync(DeleteSnapshotsOption.None);

// 辅助方法：等待复制完成
private async Task WaitForCopyCompletion(BlobClient blobClient)
{
    BlobProperties properties;
    do
    {
        properties = await blobClient.GetPropertiesAsync();
        if (properties.CopyStatus == CopyStatus.Success) break;
        await Task.Delay(1000);
    } while (true);
}

场景2：处理日志后重新上传压缩文件 #

如果需要修改日志内容，必须将处理后的明文重新压缩再上传，同时注意编码统一：

using (Stream blobStream = await blobClient.OpenReadAsync())
using (GZipStream decompressionStream = new GZipStream(blobStream, CompressionMode.Decompress))
// 明确指定UTF-8编码，避免依赖系统默认编码
using (StreamReader reader = new StreamReader(decompressionStream, Encoding.UTF8))
{
    List<string> processedLines = new List<string>();
    string line;
    while ((line = await reader.ReadLineAsync()) != null)
    {
        // 执行你的日志处理逻辑
        JObject json = JObject.Parse(line);
        string dateOfEvent = System.Convert.ToString(json["Datetime"]);
        string datePattern = "M/d/yyyy h:m:s tt";
        DateTime parsedDate;

        if (DateTime.TryParseExact(dateOfEvent, datePattern, null, DateTimeStyles.None, out parsedDate))
        {
            json["Datetime"] = parsedDate.ToString("yyyy-MM-ddTHH:mm:ssZ", CultureInfo.InvariantCulture);
        }
        processedLines.Add(json.ToString());

        // 执行PostData等其他逻辑
        var datestring = DateTime.UtcNow.ToString("r");
        var jsonBytes = Encoding.UTF8.GetBytes(json.ToString());
        string stringToHash = "POST\n" + jsonBytes.Length + "\napplication/json\n" + "x-ms-date:" + datestring + "\n/api/logs";
        string hashedString = BuildSignature(stringToHash, sharedKey);
        string signature = "SharedKey " + customerId + ":" + hashedString;
        PostData(signature, datestring, json.ToString());
    }

    // 将处理后的内容重新压缩并上传
    BlobClient destinationBlobClient = destinationBlobContainerClient.GetBlobClient(blob.Name);
    using (MemoryStream ms = new MemoryStream())
    using (GZipStream compressionStream = new GZipStream(ms, CompressionMode.Compress, true))
    using (StreamWriter writer = new StreamWriter(compressionStream, Encoding.UTF8))
    {
        foreach (string processedLine in processedLines)
        {
            await writer.WriteLineAsync(processedLine);
        }
        await writer.FlushAsync();
        await compressionStream.FlushAsync();
        ms.Position = 0;
        // 设置正确的Content-Type，确保文件被识别为压缩格式
        await destinationBlobClient.UploadAsync(ms, new BlobHttpHeaders { ContentType = "application/gzip" });
    }
}
await blobClient.DeleteAsync(DeleteSnapshotsOption.None);

关键注意事项 #

• 编码统一：始终明确使用UTF-8（或日志实际编码），禁止使用Encoding.Default，避免不同环境下编码不一致导致乱码。
• 流操作正确：不要在读取解压缩流后复用原始压缩流上传，要么直接复制原始Blob，要么重新生成压缩文件。
• Content-Type设置：上传压缩文件时必须指定application/gzip，否则客户端无法正确识别文件类型，打开时显示乱码。

内容的提问来源于stack exchange，提问作者NutsAndBolts