如何并行执行含if语句的Bash循环以加速图片转PDF处理

我有一个用于处理图片并转换为PDF的Bash函数pdf_compile，想要并行执行函数内的for循环来提升处理速度，但尝试了一种并行循环写法后没有达到预期的提速效果，以下是具体问题和解决方法：

原函数代码 #

pdf_compile() {
  start=`date +%s.%N`
  tmpdir=".temp"
  if [ ! -d "$tmpdir" ]; then
    mkdir $tmpdir
  fi
  echo "width: $1px"
  [[ -z $2 ]] && filter="sc1" || filter="$2" 
  echo "filter:" $filter
  count=$(ls | grep .jpg | wc -l)
  dn=1
  for f in *.jpg ; do
    nm="$tmpdir/$1-$filter-$f"
    if [ ! -f "$nm" ]; then
      echo "processing $f ($dn/$count)"
      if [[ "$filter" =~ ^sc1 ]]; then
        convert $f -resize $1x \
        -brightness-contrast 21x33 \
        -enhance -despeckle -sharpen 2 $nm
      elif [[ "$filter" =~ ^sc2 ]]; then
        convert $f -resize $1x \
          -brightness-contrast 19x45 \
          -sharpen 1 -despeckle -despeckle \
          -enhance $nm
      elif [[ "$filter" =~ ^sc3 ]]; then
        convert -resize $1x \
          -brightness-contrast 19x35 \
          -sharpen 1 -enhance -despeckle \
          -gamma 1.05 \
          -brightness-contrast -30x90 \
          -enhance $nm
      fi
      ffmpeg -i $nm $nm -y -hide_banner -loglevel error
    else
      echo "using cached $f ($dn/$count)"
    fi
    ((dn++))
  done
  n="$(basename $PWD)_$1px.pdf"
  echo "\nconverting to pdf"
  convert $tmpdir/$1-$filter-*.jpg $n
    if [[ "$3" =~ ^y ]]; then
      echo "removing temp"
      rm $tmpdir -r
    fi
  end=`date +%s.%N`
  runtime=$( echo "$end - $start" | bc -l )
  echo "completed in" $runtime "s"
}

尝试过的并行写法（无效） #

for f in *.jpg ; do
    ((i=i%N)); ((i++==0)) && wait
       # 原处理逻辑
done

问题分析 #

之前的并行写法无效，主要原因有几点：

1. 没有将处理任务放入后台执行（缺少&），本质还是串行运行
2. 未合理设置并发数N（比如未匹配CPU核心数），即使并行也提效有限
3. 原函数中sc3分支的convert命令缺失输入文件$f，属于代码bug，会导致该分支处理失败
4. 全局变量dn在并行场景下会被多个进程同时修改，导致计数混乱，也会干扰执行逻辑

并行优化方案 #

方案一：用xargs实现并行（推荐，简洁稳定） #

将单文件处理逻辑抽成内部函数，借助xargs -P控制并发数，自动适配CPU核心数：

pdf_compile() {
  start=$(date +%s.%N)
  tmpdir=".temp"
  mkdir -p "$tmpdir"
  local width="$1"
  local filter="${2:-sc1}"
  local clean_temp="${3:-n}"
  echo "width: ${width}px"
  echo "filter: $filter"

  # 单文件处理逻辑（修正sc3分支的输入文件bug）
  process_file() {
    local f="$1"
    local nm="${tmpdir}/${width}-${filter}-${f}"
    if [ ! -f "$nm" ]; then
      echo "processing $f"
      case "$filter" in
        sc1*)
          convert "$f" -resize "${width}x" \
            -brightness-contrast 21x33 \
            -enhance -despeckle -sharpen 2 "$nm"
          ;;
        sc2*)
          convert "$f" -resize "${width}x" \
            -brightness-contrast 19x45 \
            -sharpen 1 -despeckle -despeckle \
            -enhance "$nm"
          ;;
        sc3*)
          convert "$f" -resize "${width}x" \
            -brightness-contrast 19x35 \
            -sharpen 1 -enhance -despeckle \
            -gamma 1.05 \
            -brightness-contrast -30x90 \
            -enhance "$nm"
          ;;
      esac
      ffmpeg -i "$nm" "$nm" -y -hide_banner -loglevel error
    else
      echo "using cached $f"
    fi
  }
  export -f process_file
  export tmpdir width filter

  # 获取CPU核心数作为并发数
  local concurrency=$(nproc)
  shopt -s nullglob
  # 并行处理所有jpg文件
  printf "%s\n" *.jpg | xargs -P "$concurrency" -I {} bash -c 'process_file "$@"' _ {}

  local count=$(ls -1 *.jpg 2>/dev/null | wc -l)
  echo -e "\n共处理 $count 个文件"

  n="$(basename "$PWD")_${width}px.pdf"
  echo "converting to pdf"
  convert "${tmpdir}/${width}-${filter}-"*.jpg "$n"

  if [[ "$clean_temp" =~ ^y ]]; then
    echo "removing temp"
    rm -r "$tmpdir"
  fi

  end=$(date +%s.%N)
  runtime=$(echo "$end - $start" | bc -l)
  echo "completed in $runtime s"
}

方案二：改进后台循环控制并发数 #

通过后台进程+等待逻辑手动控制并发数，保留原有的计数提示：

pdf_compile() {
  start=$(date +%s.%N)
  tmpdir=".temp"
  mkdir -p "$tmpdir"
  local width="$1"
  local filter="${2:-sc1}"
  local clean_temp="${3:-n}"
  echo "width: ${width}px"
  echo "filter: $filter"

  # 获取CPU核心数作为并发数
  local concurrency=$(nproc)
  local running=0
  shopt -s nullglob
  local files=(*.jpg)
  local count=${#files[@]}
  local dn=1

  for f in "${files[@]}"; do
    # 达到并发上限时，等待任意后台进程结束
    while (( running >= concurrency )); do
      wait -n
      (( running-- ))
    done

    # 后台执行单文件处理
    (
      local nm="${tmpdir}/${width}-${filter}-${f}"
      if [ ! -f "$nm" ]; then
        echo "processing $f ($dn/$count)"
        case "$filter" in
          sc1*)
            convert "$f" -resize "${width}x" \
              -brightness-contrast 21x33 \
              -enhance -despeckle -sharpen 2 "$nm"
            ;;
          sc2*)
            convert "$f" -resize "${width}x" \
              -brightness-contrast 19x45 \
              -sharpen 1 -despeckle -despeckle \
              -enhance "$nm"
            ;;
          sc3*)
            convert "$f" -resize "${width}x" \
              -brightness-contrast 19x35 \
              -sharpen 1 -enhance -despeckle \
              -gamma 1.05 \
              -brightness-contrast -30x90 \
              -enhance "$nm"
            ;;
        esac
        ffmpeg -i "$nm" "$nm" -y -hide_banner -loglevel error
      else
        echo "using cached $f ($dn/$count)"
      fi
    ) &
    (( running++ ))
    (( dn++ ))
  done

  # 等待所有后台进程完成
  wait

  n="$(basename "$PWD")_${width}px.pdf"
  echo -e "\nconverting to pdf"
  convert "${tmpdir}/${width}-${filter}-"*.jpg "$n"

  if [[ "$clean_temp" =~ ^y ]]; then
    echo "removing temp"
    rm -r "$tmpdir"
  fi

  end=$(date +%s.%N)
  runtime=$(echo "$end - $start" | bc -l)
  echo "completed in $runtime s"
}

内容的提问来源于stack exchange，提问作者Roopesh