解决Shell脚本生成CSV文件表头与数据间的空行问题

调用autorep命令获取作业数据并转换为CSV格式的Shell脚本，存在表头与第一行数据之间出现空值行的问题，调整awk中的NR条件无法解决该问题。

脚本代码 #

#!/bin/bash

read -p "Enter /DIR/PATH/FILENAME where you wish to copy the data: " FILENAME
echo "Enter the JOB_NAME or %SEARCHSTRING%"


while read -r i;
do

awk '
BEGIN {
print "\"insert_job\",\"job_type\",\"box_name\",\"command\",\"machine\",\"owner\",\"date_conditions\",\"condition\",\"run_calendar\",\"exclude_calendar\",\"days_of_week\",\"run_window\",\"start_times\",\"start_mins\",\"resources\",\"profile\",\"term_run_time\",\"watch_file\",\"watch_interval\"" }

/job_type/ {
    if (NR>1){printf "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n", jn, jt, box, cmd, mcn, own, dc, c, rc, ec, dow, ruw, st, sm, res, prof, trt, wf, wi} jn="\""$2"\""; jt="\""$4"\""; box="\" \""; cmd="\" \""; mcn="\" \""; own="\" \""; dc="\" \""; c="\" \""; rc="\" \""; ec="\" \""; dow="\" \""; ruw="\" \""; st="\" \""; sm="\" \""; res="\" \""; prof="\" \""; trt="\" \""; wf="\" \""; wi="\" \""}
/box_name/ {box="\""$2"\""}
/command/ {$0=substr($0,index($0,$2)); cmd="\""$0"\""}
/machine/ {mcn="\""$2"\""}
/owner/   {own="\""$2"\""}
/date_conditions/ {dc="\""$2"\""}
/condition/ {$0=substr($0,index($0,$2)); c="\""$0"\""}
/run_calendar/ {rc="\""$2"\""}
/exclude_calendar/ {ec="\""$2"\""}
/days_of_week/ {dow="\""$2"\""}
/run_window/ {ruw="\""$2"\""}
/start_times/ {gsub("\"","",$2); st="\""$2"\""}
/^start_mins/ {sm="\""$2"\""}
/profile/ {prof="\""$2"\""}
/term_run_time/ {trt="\""$2"\""}
/watch_file/ {wf="\""$2"\""}
/watch_interval/ {wi="\""$2"\""}
/resources/ {res="\""$2"\""}
END{printf "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n", jn, jt, box, cmd, mcn, own, dc, c, rc, ec, dow, ruw, st, sm, res, prof, trt, wf, wi}
' < <(autorep -j $i -q) > $FILENAME.csv

break
done

测试数据 #

/* ----------------- Test_A ----------------- */

insert_job: Test_A  job_type: CMD
command: sleep 3000
machine: machine1
owner: user1
permission:
date_conditions: 1
days_of_week: mo,tu,we,th,FR
start_times: “06:00,08:00,10:00,12:00,14:00,16:00”
term_run_time: 1
alarm_if_fail: 1
alarm_if_terminated: 1


/* ----------------- Test_B ----------------- */

insert_job: Test_B    job_type: CMD
command: echo
machine: machine2
owner: user2
permission:
date_conditions: 0
description: "Test"
std_out_file: "/tmp/$AUTO_JOB_NAME.$AUTORUN.out"
std_err_file: "/tmp/$AUTO_JOB_NAME.$AUTORUN.err"
max_run_alarm: 1
alarm_if_fail: 0
alarm_if_terminated: 0
send_notification: 1



/* ----------------- Test_c ----------------- */

insert_job: Test_c   job_type: CMD
command: sleep 10
machine: machine3
owner: user3
permission:
date_conditions: 0
alarm_if_fail: 0
alarm_if_terminated: 0


/* ----------------- Test_d ----------------- */

insert_job: Test_d   job_type: CMD
command: ls
machine: machine4
owner: user4
permission:
date_conditions: 0
alarm_if_fail: 1
alarm_if_terminated: 1

当前错误输出 #

"insert_job","job_type","box_name","command","machine","owner","date_conditions","condition","run_calendar","exclude_calendar","days_of_week","run_window","start_times","start_mins","resources","profile","term_run_time","watch_file","watch_interval"
,,,,,,,,,,,,,,,,,,
"Test_A","CMD"," ","sleep 3000","machine1","user1","0","0"," "," "," "," "," "," "," "," ","1"," "," "
"Test_B","CMD"," ","echo","machine2","user2","0","0"," "," "," "," "," "," "," "," "," "," "," "
"Test_c","CMD"," ","sleep 10","machine3","user3","0","0"," "," "," "," "," "," "," "," "," "," "," "
"Test_d","CMD"," ","ls","machine4","user4","0","0"," "," "," "," "," "," "," "," "," "," "," "

期望输出 #

"insert_job","job_type","box_name","command","machine","owner","date_conditions","condition","run_calendar","exclude_calendar","days_of_week","run_window","start_times","start_mins","resources","profile","term_run_time","watch_file","watch_interval"
"Test_A","CMD"," ","sleep 3000","machine1","user1","0","0"," "," "," "," "," "," "," "," ","1"," "," "
"Test_B","CMD"," ","echo","machine2","user2","0","0"," "," "," "," "," "," "," "," "," "," "," "
"Test_c","CMD"," ","sleep 10","machine3","user3","0","0"," "," "," "," "," "," "," "," "," "," "," "
"Test_d","CMD"," ","ls","machine4","user4","0","0"," "," "," "," "," "," "," "," "," "," "," "

问题根源是awk脚本中用NR>1作为打印判断条件，而第一个匹配/job_type/的行NR值大于1（测试数据中该行前有注释和空行），此时jn等变量未赋值，导致打印空行。

修改awk的判断逻辑，用标记变量控制首次打印：

#!/bin/bash

read -p "Enter /DIR/PATH/FILENAME where you wish to copy the data: " FILENAME
echo "Enter the JOB_NAME or %SEARCHSTRING%"


while read -r i;
do

awk '
BEGIN {
print "\"insert_job\",\"job_type\",\"box_name\",\"command\",\"machine\",\"owner\",\"date_conditions\",\"condition\",\"run_calendar\",\"exclude_calendar\",\"days_of_week\",\"run_window\",\"start_times\",\"start_mins\",\"resources\",\"profile\",\"term_run_time\",\"watch_file\",\"watch_interval\"";
first_job = 1;  # 标记是否为第一个作业
}

/job_type/ {
    if (!first_job) {  # 非第一个作业时才打印上一个作业的数据
        printf "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n", jn, jt, box, cmd, mcn, own, dc, c, rc, ec, dow, ruw, st, sm, res, prof, trt, wf, wi
    } 
    # 重置并赋值当前作业字段
    jn="\""$2"\""; 
    jt="\""$4"\""; 
    box="\" \""; 
    cmd="\" \""; 
    mcn="\" \""; 
    own="\" \""; 
    dc="\" \""; 
    c="\" \""; 
    rc="\" \""; 
    ec="\" \""; 
    dow="\" \""; 
    ruw="\" \""; 
    st="\" \""; 
    sm="\" \""; 
    res="\" \""; 
    prof="\" \""; 
    trt="\" \""; 
    wf="\" \""; 
    wi="\" \"";
    first_job = 0;  # 第一个作业处理完成，切换标记
}
/box_name/ {box="\""$2"\""}
/command/ {$0=substr($0,index($0,$2)); cmd="\""$0"\""}
/machine/ {mcn="\""$2"\""}
/owner/   {own="\""$2"\""}
/date_conditions/ {dc="\""$2"\""}
/condition/ {$0=substr($0,index($0,$2)); c="\""$0"\""}
/run_calendar/ {rc="\""$2"\""}
/exclude_calendar/ {ec="\""$2"\""}
/days_of_week/ {dow="\""$2"\""}
/run_window/ {ruw="\""$2"\""}
/start_times/ {gsub("\"","",$2); st="\""$2"\""}
/^start_mins/ {sm="\""$2"\""}
/profile/ {prof="\""$2"\""}
/term_run_time/ {trt="\""$2"\""}
/watch_file/ {wf="\""$2"\""}
/watch_interval/ {wi="\""$2"\""}
/resources/ {res="\""$2"\""}
END{printf "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n", jn, jt, box, cmd, mcn, own, dc, c, rc, ec, dow, ruw, st, sm, res, prof, trt, wf, wi}
' < <(autorep -j $i -q) > $FILENAME.csv

break
done

修复说明 #

1. 在BEGIN块中新增first_job标记，初始值为1，表示当前是第一个作业。
2. 将/job_type/块中的if (NR>1)改为if (!first_job)，只有处理非第一个作业时，才打印上一个作业的缓存数据，避免首次触发时打印空值。
3. 赋值当前作业字段后，将first_job设为0，后续作业触发时正常打印上一条数据。

此修改可彻底消除表头与第一行数据之间的空行，符合期望输出格式。

内容的提问来源于stack exchange，提问作者NecroCoder