AWS SDK .NET ListObjectsV2响应解组错误排查求助

错误现象 #

• 存储桶根目录对象查询正常，但查询子目录时触发两个错误：
  • Error unmarshalling response back from AWS. HTTP Status Code: 200 OK
  • Root element is missing
• 存储桶根目录对象数量超1000，需通过子目录筛选目标内容

环境信息 #

• AWS SDK .NET 4.5
• SSIS脚本任务，Visual Basic 2019编译为.NET 4.8

可复现代码 #

Imports System
Imports System.Data
Imports System.Math
Imports Microsoft.SqlServer.Dts.Runtime
Imports Amazon.S3
Imports Amazon.S3.Model
Imports Amazon.Runtime
Imports Amazon
Imports Amazon.S3.Util
Imports System.Collections.ObjectModel
Imports System.IO

'ScriptMain is the entry point class of the script.  Do not change the name, attributes,
'or parent of this class.
<Microsoft.SqlServer.Dts.Tasks.ScriptTask.SSISScriptTaskEntryPointAttribute()>
<System.CLSCompliantAttribute(False)>
Partial Public Class ScriptMain
    Inherits Microsoft.SqlServer.Dts.Tasks.ScriptTask.VSTARTScriptObjectModelBase

    Public Sub Main()
        Dim filecol As ObservableCollection(Of String)
        Try
            'filecol = ListingFiles("/gov-fpac-rma-pubfs-production/pub/References/actuarial_data_master/2023/")
            'filecol = ListingFiles("/gov-fpac-rma-pubfs-production/") 'Bucket root
            filecol = ListingFiles("/gov-fpac-rma-pubfs-production/pub/")

            Dts.TaskResult = ScriptResults.Success
        Catch ex As Exception
            Console.WriteLine(ex.Message.ToString)
            Dts.TaskResult = ScriptResults.Failure
        End Try
    End Sub

#Region "ScriptResults declaration"
    Enum ScriptResults
        Success = Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Success
        Failure = Microsoft.SqlServer.Dts.Runtime.DTSExecResult.Failure
    End Enum
#End Region

    Private Function ListingFiles(bucketName As String, Optional foldername As String = "/") As ObservableCollection(Of String)
        Dim obsv As New ObservableCollection(Of String)
        Dim delimiter As String = "/"
        Dim AWS_ACCESS_KEY As String = "xxxxxxxxxxxx" 'Add your Access Key here
        Dim AWS_SECRET_KEY As String = "xxxxxxxxxxxxxxxxx" ' 'Add your Secret here

        Dim s3config As AmazonS3Config = New AmazonS3Config
        With s3config
            .ForcePathStyle = True
            .RegionEndpoint = RegionEndpoint.USEast1
        End With

        Dim s3Client As AmazonS3Client = New AmazonS3Client(AWS_ACCESS_KEY, AWS_SECRET_KEY, s3config)

        If Not foldername.EndsWith(delimiter) Then
            foldername = String.Format("{0}{1}", foldername, delimiter)
        End If
        Try
            Try
                Dim request As New ListObjectsV2Request()
                With request
                    .BucketName = bucketName
                End With

                Do
                    Dim response As New ListObjectsV2Response()
                    response = s3Client.ListObjectsV2(request)

                    For i As Integer = 1 To response.S3Objects.Count - 1
                        Dim entry As S3Object = response.S3Objects(i)
                        If Not foldername = "/" Then
                            If entry.Key.ToString.StartsWith(foldername) Then
                                Dim replacementstring As String = Replace(entry.Key, foldername, "")
                                If Not replacementstring = "" Then
                                    obsv.Add(replacementstring)
                                End If
                            End If
                        Else
                            obsv.Add(Replace(entry.Key, foldername, ""))
                        End If
                        MessageBox.Show(entry.Key + "  " + entry.LastModified.ToString())
                    Next
                    If (response.IsTruncated) Then
                        request.ContinuationToken = response.NextContinuationToken
                    Else
                        request = Nothing
                    End If
                Loop Until IsNothing(request)
            Catch ex As AmazonS3Exception
                Console.WriteLine(ex.Message.ToString)
                Dts.TaskResult = ScriptResults.Failure
            End Try
        Catch ex As Exception
            Console.WriteLine(ex.Message.ToString)
            Dts.TaskResult = ScriptResults.Failure
        End Try
        Return obsv
    End Function
End Class

问题原因及解决方法 #

1. BucketName参数格式错误 #

当前代码将包含路径前缀的完整字符串（如/gov-fpac-rma-pubfs-production/pub/）传入BucketName，但S3的BucketName仅需纯存储桶名称，路径应通过Prefix参数指定。错误的桶名会导致AWS返回异常响应格式，触发反序列化错误。

修复方法：
拆分存储桶名称和前缀，调整ListingFiles方法的调用与实现：

• 调用时传入纯存储桶名称，前缀作为第二个参数
• 在ListObjectsV2Request中设置Prefix属性过滤子目录，无需后续通过StartsWith二次筛选

修改后Main方法调用示例：

filecol = ListingFiles("gov-fpac-rma-pubfs-production", "pub/")

修改后ListingFiles核心逻辑：

Private Function ListingFiles(bucketName As String, Optional prefix As String = "") As ObservableCollection(Of String)
    Dim obsv As New ObservableCollection(Of String)
    Dim delimiter As String = "/"
    ' ... 密钥、配置初始化代码不变

    Try
        Dim request As New ListObjectsV2Request()
        With request
            .BucketName = bucketName
            If Not String.IsNullOrEmpty(prefix) Then
                .Prefix = prefix
            End If
        End With

        Do
            Dim response As ListObjectsV2Response = s3Client.ListObjectsV2(request)

            ' 直接遍历返回的对象，无需二次过滤
            For Each entry As S3Object In response.S3Objects
                Dim objKey As String = entry.Key.Substring(prefix.Length)
                If Not String.IsNullOrEmpty(objKey) Then
                    obsv.Add(objKey)
                End If
                MessageBox.Show(entry.Key + "  " + entry.LastModified.ToString())
            Next

            If response.IsTruncated Then
                request.ContinuationToken = response.NextContinuationToken
            Else
                request = Nothing
            End If
        Loop Until request Is Nothing
    ' ... 异常处理代码不变
End Function

2. 索引遍历遗漏数据 #

原代码使用For i As Integer = 1 To response.S3Objects.Count - 1，跳过了索引0的对象，导致数据遗漏。应改为从0开始遍历：

For i As Integer = 0 To response.S3Objects.Count - 1
' 或更简洁的For Each遍历
For Each entry As S3Object In response.S3Objects

3. SDK版本兼容性问题 #

AWS SDK .NET 4.5与.NET 4.8存在潜在兼容性问题，建议升级到AWSSDK.S3 v3.5.x及以上版本（兼容.NET 4.5+），确保反序列化逻辑匹配AWS最新响应格式。

内容的提问来源于stack exchange，提问作者CStrom