大量文件目录下PowerShell搜索函数运行缓慢的优化咨询

我在指定Region文件夹内按规则搜索文件并展示，但当目录包含大量文件（如200k个）时，搜索耗时约10分钟，文件量少时速度正常。尝试使用Runspaces优化但未成功，耗时无变化。当前使用PowerShell版本5.1.22621.4249，以下是我的初始实现代码、尝试Runspaces的代码及已实现缓存功能的代码，寻求有效的搜索提速方案：

初始实现代码 #

function Convert-FileName { 
    param (
        [string]$fileName
    )

    $fileName = $fileName.TrimStart('.')

    if ($fileName -match "^(\d+)\.([A-Z]+)\.01\.(\d+)\.(\d{8})$") {
        return @{
            "PayerID"    = $matches[1]
            "Region"     = $matches[2]
            "CheckNum"   = $matches[3]
            "CheckDate"  = [datetime]::ParseExact($matches[4], "yyyyMMdd", $null)
        }
    } else {
        return $null
    }
}

$buttonSearch.Add_Click({
    $region = $comboBoxRegion.Text
    $payIDRange = $textBoxPayID.Text
    $checkNumRange = $textBoxCheckNum.Text
    $checkDateRange = $textBoxCheckDate.Text

    if ([string]::IsNullOrWhiteSpace($global:selectedFolder)) {
        [System.Windows.Forms.MessageBox]::Show("Please choose a folder before searching.")
        return
    }

    if ([string]::IsNullOrWhiteSpace($region)) {
        $dataGridView.Rows.Clear()
        [System.Windows.Forms.MessageBox]::Show("Please select a region.")
        return
    }

    if ([string]::IsNullOrWhiteSpace($payIDRange) -or $payIDRange -notmatch "^\d+-\d+$") {
        $dataGridView.Rows.Clear()
        [System.Windows.Forms.MessageBox]::Show("Invalid Pay ID range.")
        return
    }

    $global:totalResults = New-Object System.Collections.Generic.List[object]

    $payIDStart, $payIDEnd = $payIDRange -split "-", 2
    $payIDStart = [int]$payIDStart
    $payIDEnd = [int]$payIDEnd

    $allFiles = Get-ChildItem -Path "$global:selectedFolder\$region" -File 

    foreach ($file in $allFiles) {
        $fileInfo = Convert-FileName -fileName $file.Name
        if ($fileInfo -ne $null) {
            $extractedPayerID = [int]$fileInfo.PayerID
            $payIDValid = $extractedPayerID -ge $payIDStart -and $extractedPayerID -le $payIDEnd

            $checkNumValid = $true
            if ($checkNumRange -ne "") {
                $checkNumStart, $checkNumEnd = $checkNumRange -split "-", 2
                $checkNumValid = $fileInfo.CheckNum -ge [int]$checkNumStart -and $fileInfo.CheckNum -le [int]$checkNumEnd
            }

            $checkDateValid = $true
            if ($checkDateRange -ne "") {
                $startDate, $endDate = $checkDateRange -split " - ", 2
                $startDate = [datetime]::ParseExact($startDate.Trim(), "yyyy-MM-dd", $null)
                $endDate = [datetime]::ParseExact($endDate.Trim(), "yyyy-MM-dd", $null)
                $checkDateValid = $fileInfo.CheckDate -ge $startDate -and $fileInfo.CheckDate -le $endDate
            }

            if ($payIDValid -and $checkNumValid -and $checkDateValid) {
                $resultObject = New-Object -TypeName PSObject -Property @{
                    Name = $file.Name
                    Path = $file.FullName
                }
                $global:totalResults.Add($resultObject)
            }
        }
    }

    if ($global:totalResults.Count -eq 0) {
        $dataGridView.Rows.Clear()
        [System.Windows.Forms.MessageBox]::Show("No results found.")
        return
    }

    $global:currentPage = 0
    Show-Results
})

尝试Runspaces优化的代码 #

function Search-Files {
    Param(
        [string]$selectedFolder,
        [string]$region,
        [string]$payIDRange,
        [string]$checkNumRange,
        [string]$checkDateRange
    )

    $payIDStart, $payIDEnd = $payIDRange -split "-"
    $payIDStart = [int]$payIDStart
    $payIDEnd = [int]$payIDEnd

    $allFiles = Get-ChildItem -Path "$selectedFolder\$region" -File

    $file_count = $allFiles.Count
    $cpu_count = [Environment]::ProcessorCount
    $optimal_runspaces = [Math]::Min($cpu_count, $file_count)
    $file_chunks = [System.Collections.Generic.List[object[]]]::new($optimal_runspaces)

    $runspace_pool = [runspacefactory]::CreateRunspacePool(1, $optimal_runspaces)
    $runspace_pool.Open()

    $chunk_size = [Math]::Ceiling($file_count / $optimal_runspaces)
    for ($i = 0; $i -lt $optimal_runspaces; $i++) {
        $start = $i * $chunk_size
        $end = [Math]::Min(($start + $chunk_size - 1), ($file_count - 1))

        $file_chunks.Add($allFiles[$start..$end])
    }

    $scriptblock = {
        Param($files, $payIDStart, $payIDEnd, $checkNumRange, $checkDateRange)
        function Convert-FileName { 
            param (
                [string]$fileName
            )
        
            $fileName = $fileName.TrimStart('.')
        
            if ($fileName -match "^(\d+)\.([A-Z]+)\.01\.(\d+)\.(\d{8})$") {
                return @{
                    "PayerID"    = $matches[1]
                    "Region"     = $matches[2]
                    "CheckNum"   = $matches[3]
                    "CheckDate"  = [datetime]::ParseExact($matches[4], "yyyyMMdd", $null)
                }
            } else {
                return $null
            }
        }

        $results = [System.Collections.Generic.List[object]]::new()
        foreach ($file in $files) {
            $fileInfo = Convert-FileName $file.Name
            if ($fileInfo -eq $null) { continue }

            $extractedPayerID = [int]$fileInfo.PayerID.Trim()
            if ($extractedPayerID -lt $payIDStart -or $extractedPayerID -gt $payIDEnd) { continue }

            $checkNumValid = $true
            if ($checkNumRange -ne "") {
                $checkNumStart, $checkNumEnd = $checkNumRange -split "-"
                $checkNumValid = ($fileInfo.CheckNum -ge [int]$checkNumStart -and $fileInfo.CheckNum -le [int]$checkNumEnd)
            }

            $checkDateValid = $true
            if ($checkDateRange -ne "") {
                $startDate, $endDate = $checkDateRange -split " - "
                $startDate = [datetime]::Parse($startDate)
                $endDate = [datetime]::Parse($endDate)
                $checkDateValid = ($fileInfo.CheckDate -ge $startDate -and $fileInfo.CheckDate -le $endDate)
            }

            if ($checkNumValid -and $checkDateValid) {
                $results.Add($file)
            }
        }
        return $results
    }

    $runspaces = @{}
    foreach ($chunk in $file_chunks) {
        $runspace = [powershell]::Create().AddScript($scriptblock).AddArgument($chunk).AddArgument($payIDStart).AddArgument($payIDEnd).AddArgument($checkNumRange).AddArgument($checkDateRange)
        $runspace.RunspacePool = $runspace_pool
        $runspaces[$runspace] = $runspace.BeginInvoke()
    }

    while ($runspaces.Values | Where-Object { -not $_.IsCompleted }) {
        Start-Sleep -Milliseconds 100
    }

    $all_results = @()
    foreach ($runspace in $runspaces.Keys) {
        $all_results += $runspace.EndInvoke($runspaces[$runspace])
        $runspace.Dispose()
    }

    $runspace_pool.Close()
    $runspace_pool.Dispose()

    return $all_results
}

$buttonSearch.Add_Click({
    $region = $comboBoxRegion.Text
    $payIDRange = $textBoxPayID.Text
    $checkNumRange = $textBoxCheckNum.Text
    $checkDateRange = $textBoxCheckDate.Text

    if ($global:selectedFolder -eq "") {
        [System.Windows.Forms.MessageBox]::Show("Please choose a folder before searching.")
        return
    }

    if ($region -eq "") {
        $dataGridView.Rows.Clear()
        [System.Windows.Forms.MessageBox]::Show("Please select a region.")
        return
    }

    if ($payIDRange -eq "" -or $payIDRange -notmatch "^\d+-\d+$") {
        $dataGridView.Rows.Clear()
        [System.Windows.Forms.MessageBox]::Show("Invalid Pay ID range.")
        return
    }

    $results = Search-Files -selectedFolder $global:selectedFolder -region $region -payIDRange $payIDRange -checkNumRange $checkNumRange -checkDateRange $checkDateRange

    if ($results.Count -eq 0) {
        $dataGridView.Rows.Clear()
        [System.Windows.Forms.MessageBox]::Show("No results found.")
        return
    }

    $global:totalResults = $results

    $global:currentPage = 0
    Show-Results
})

已实现缓存功能的代码 #

$global:Cache = @{}
$buttonSearch.Add_Click({
    $region = $comboBoxRegion.Text
    $payIDRange = $textBoxPayID.Text
    $checkNumRange = $textBoxCheckNum.Text
    $checkDateRange = $textBoxCheckDate.Text

    if ([string]::IsNullOrWhiteSpace($global:selectedFolder)) {
        [System.Windows.Forms.MessageBox]::Show("Please choose a folder before searching.")
        return
    }

    if ([string]::IsNullOrWhiteSpace($region)) {
        $dataGridView.Rows.Clear()
        [System.Windows.Forms.MessageBox]::Show("Please select a region.")
        return
    }

    if ([string]::IsNullOrWhiteSpace($payIDRange) -or $payIDRange -notmatch "^\d+-\d+$") {
        $dataGridView.Rows.Clear()
        [System.Windows.Forms.MessageBox]::Show("Invalid Pay ID range.")
        return
    }

    $global:totalResults = New-Object System.Collections.Generic.List[object]

    $payIDStart, $payIDEnd = $payIDRange -split "-", 2
    $payIDStart = [int]$payIDStart
    $payIDEnd = [int]$payIDEnd

    $Directory = $global:selectedFolder + '\' + $region
    $allFiles = [System.IO.Directory]::EnumerateFiles($Directory, '*', [System.IO.SearchOption]::AllDirectories)
    $startDate = $null
    $endDate = $null
    
    if (![string]::IsNullOrWhiteSpace($checkDateRange)) {
        $dateParts = $checkDateRange -split " - ", 2
        if ($dateParts.Length -eq 2) {
            try {
                $startDate = [datetime]::ParseExact($dateParts[0].Trim(), "yyyy-MM-dd", $null)
                $endDate = [datetime]::ParseExact($dateParts[1].Trim(), "yyyy-MM-dd", $null)
            } catch {
                Write-Host "Error parsing dates: '$($dateParts[0].Trim())' or '$($dateParts[1].Trim())'. Please ensure they are in 'yyyy-MM-dd' format."
                return
            }
        }
    }

    foreach ($filePath in $allFiles) {
        $file = New-Object System.IO.FileInfo $filePath
        $currentCacheKey = Generate-CacheKey -fileName $file.FullName -payIDRange $payIDRange -checkNumRange $checkNumRange -checkDateRange $checkDateRange
        
        if (-not $global:Cache.ContainsKey($currentCacheKey)) {
            $fileInfo = Convert-FileName -fileName $file.Name
            if ($fileInfo -ne $null) {
                $extractedPayerID = [int]$fileInfo.PayerID
                $payIDValid = $extractedPayerID -ge $payIDStart -and $extractedPayerID -le $payIDEnd
    
                $checkNumValid = $true
                if ($checkNumRange -ne "") {
                    $checkNumStart, $checkNumEnd = $checkNumRange -split "-", 2
                    $checkNumValid = $fileInfo.CheckNum -ge [int]$checkNumStart -and $fileInfo.CheckNum -le [int]$checkNumEnd
                }
    
                $checkDateValid = $true
                if ($checkDateRange -ne "") {
                    $checkDateValid = $fileInfo.CheckDate -ge $startDate -and $fileInfo.CheckDate -le $endDate
                }
    
                if ($payIDValid -and $checkNumValid -and $checkDateValid) {
                    $resultObject = [PSCustomObject] @{
                        Name = $file.Name
                        Path = $file.FullName
                    }
                    $global:totalResults.Add($resultObject)
                    $global:Cache[$currentCacheKey] = $resultObject
                } else {
                    $global:Cache[$currentCacheKey] = $null
                }
            } else {
                $global:Cache[$currentCacheKey] = $null
            }
        } else {
            $resultObject = $global:Cache[$currentCacheKey]
            if ($resultObject) {
                $global:totalResults.Add($resultObject)
            }
        }
    }

    if ($global:totalResults.Count -eq 0) {
        $dataGridView.Rows.Clear()
        [System.Windows.Forms.MessageBox]::Show("No results found.")
        return
    }

    $global:currentPage = 0
    Show-Results
})

内容的提问来源于stack exchange，提问作者k5927211