如何遍历JSON文件中的interval并生成符合要求的DataFrame
问题说明
我正在遍历JSON文件,抽取指定字段生成符合要求的DataFrame。已经完成了代码的初步实现,但当前JSON文件结构发生了小幅调整,无法确定需要修改代码的哪一部分才能得到预期输出。
旧版JSON返回结果
queryResult: {'results': [{'data': [{'interval': '2021-10-11T11:46:25.000Z/2021-10-18T11:49:48.000Z', 'metrics': [{'metric': 'nOffered', 'qualifier': None, 'stats': {'count': 7, 'count_negative': None, 'count_positive': None, 'current': None, 'denominator': None, 'max': None, 'min': None, 'numerator': None, 'ratio': None, 'sum': None, 'target': None}}, {'metric': 'nTransferred', 'qualifier': None, 'stats': {'count': 1, 'count_negative': None, 'count_positive': None, 'current': None, 'denominator': None, 'max': None, 'min': None, 'numerator': None, 'ratio': None, 'sum': None, 'target': None}}], 'views': None}], 'group': {'mediaType': 'voice', 'queueId': '73643cff-799b-41ae-9a67-efcf5e593155'}}]}
旧版输出DataFrame
Queue_Id,Interval Start,Interval End,nOffered_count,nOffered_sum,nOffered.denominator,nOffered.numerator,nTransferred_count,nTransferred_sum,nTransferred.denominator,nTransferred.numerator 73643cff-799b-41ae-9a67-efcf5e593155,2021-10-11T11:46:25.000Z,2021-10-18T11:49:48.000Z,7,,,,1.0,,,
新版JSON返回结果
queryResult: {'results': [{'data': [{'interval': '2021-10-11T11:46:25.000Z/2021-10-12T11:46:25.000Z', 'metrics': [{'metric': 'nOffered', 'qualifier': None, 'stats': {'count': 1, 'count_negative': None, 'count_positive': None, 'current': None, 'denominator': None, 'max': None, 'min': None, 'numerator': None, 'ratio': None, 'sum': None, 'target': None}}], 'views': None}, {'interval': '2021-10-13T11:46:25.000Z/2021-10-14T11:46:25.000Z', 'metrics': [{'metric': 'nOffered', 'qualifier': None, 'stats': {'count': 2, 'count_negative': None, 'count_positive': None, 'current': None, 'denominator': None, 'max': None, 'min': None, 'numerator': None, 'ratio': None, 'sum': None, 'target': None}}, {'metric': 'nTransferred', 'qualifier': None, 'stats': {'count': 1, 'count_negative': None, 'count_positive': None, 'current': None, 'denominator': None, 'max': None, 'min': None, 'numerator': None, 'ratio': None, 'sum': None, 'target': None}}], 'views': None}, {'interval': '2021-10-14T11:46:25.000Z/2021-10-15T11:46:25.000Z', 'metrics': [{'metric': 'nOffered', 'qualifier': None, 'stats': {'count': 3, 'count_negative': None, 'count_positive': None, 'current': None, 'denominator': None, 'max': None, 'min': None, 'numerator': None, 'ratio': None, 'sum': None, 'target': None}}], 'views': None}, {'interval': '2021-10-15T11:46:25.000Z/2021-10-16T11:46:25.000Z', 'metrics': [{'metric': 'nOffered', 'qualifier': None, 'stats': {'count': 1, 'count_negative': None, 'count_positive': None, 'current': None, 'denominator': None, 'max': None, 'min': None, 'numerator': None, 'ratio': None, 'sum': None, 'target': None}}], 'views': None}], 'group': {'mediaType': 'voice', 'queueId': '73643cff-799b-41ae-9a67-efcf5e593155'}}]}
预期新版输出DataFrame
Queue_Id,Interval Start,Interval End,nOffered_count,nOffered_sum,nOffered.denominator,nOffered.numerator,nTransferred_count,nTransferred_sum,nTransferred.denominator,nTransferred.numerator 73643cff-799b-41ae-9a67-efcf5e593155,2021-10-11T11:46:25.000Z,2021-10-12T11:46:25.000Z,1,,,,,,, 73643cff-799b-41ae-9a67-efcf5e593155,2021-10-13T11:46:25.000Z,2021-10-14T11:46:25.000Z,2,,,,1,,, 73643cff-799b-41ae-9a67-efcf5e593155,2021-10-14T11:46:25.000Z,2021-10-15T11:46:25.000Z,3,,,,,,, 73643cff-799b-41ae-9a67-efcf5e593155,2021-10-15T11:46:25.000Z,2021-10-16T11:46:25.000Z,1,,,,,,,
现有代码
column_names = [] if(query_result.results != None): for item in query_result.results: data_lst = [] for lst_data in item.data: print("####################################") print(lst_data) print("####################################") for met in lst_data.metrics: metric_name = met.metric column_names.append('Queue_Id') column_names.append(metric_name+'_count') column_names.append(metric_name+'_sum') column_names.append(metric_name+'.denominator') column_names.append(metric_name+'.numerator') column_names.append('Interval Start') column_names.append('Interval End') data_lst.append(queue_id) data_lst.append(met.stats.count) data_lst.append(met.stats.sum) data_lst.append(met.stats.denominator) data_lst.append(met.stats.numerator) data_lst.append(lst_data.interval.split('/')[0]) data_lst.append(lst_data.interval.split('/')[1]) print(data_lst) else: data_lst = [] metric_name = query.metrics[0] column_names.append('Queue_Id') column_names.append(metric_name+'_count') column_names.append(metric_name+'_sum') column_names.append(metric_name+'.denominator') column_names.append(metric_name+'.numerator') column_names.append('Interval Start') column_names.append('Interval End') data_lst.append(queue_id) data_lst.append('') data_lst.append('') data_lst.append('') data_lst.append('') data_lst.append(query.interval.split('/')[0]) data_lst.append(query.interval.split('/')[1]) print("data_lst", data_lst) print("column_names", column_names) return data_lst, column_names
修改方案
现有代码问题说明
- 列名重复生成:循环遍历每个指标时重复添加相同的列名,会导致最终列名数量错误
- 数据存储逻辑错误:旧版JSON的
data数组只有1个时间区间,新版有4个时间区间,现有代码没有按「每个时间区间对应一行数据」的逻辑拆分,且同一时间区间下的多个指标字段没有合并到同一行 - 缺失指标无默认值:部分时间区间没有
nTransferred指标,对应字段没有补空值,会导致列错位
修改后代码
# 先预定义所有需要的列名,避免重复生成 column_names = [ 'Queue_Id', 'Interval Start', 'Interval End', 'nOffered_count', 'nOffered_sum', 'nOffered.denominator', 'nOffered.numerator', 'nTransferred_count', 'nTransferred_sum', 'nTransferred.denominator', 'nTransferred.numerator' ] # 存储所有行数据的列表 all_data = [] if query_result.results is not None: for item in query_result.results: queue_id = item.group.queueId # 从group字段取队列ID,避免硬编码 for lst_data in item.data: # 初始化单行数据,默认所有字段为空 row = {col: None for col in column_names} # 填充公共字段 row['Queue_Id'] = queue_id interval_start, interval_end = lst_data.interval.split('/') row['Interval Start'] = interval_start row['Interval End'] = interval_end # 填充指标字段 for met in lst_data.metrics: metric_name = met.metric row[f'{metric_name}_count'] = met.stats.count row[f'{metric_name}_sum'] = met.stats.sum row[f'{metric_name}.denominator'] = met.stats.denominator row[f'{metric_name}.numerator'] = met.stats.numerator # 把行数据转成列表,添加到总数据列表 all_data.append([row[col] for col in column_names]) else: # 无返回结果时的逻辑,保持原有逻辑适配新列 row = {col: None for col in column_names} row['Queue_Id'] = queue_id interval_start, interval_end = query.interval.split('/') row['Interval Start'] = interval_start row['Interval End'] = interval_end all_data.append([row[col] for col in column_names]) return all_data, column_names
修改后返回的all_data是二维列表,每一个子列表对应DataFrame的一行,直接传入pd.DataFrame(all_data, columns=column_names)即可生成符合预期的结果。
内容的提问来源于stack exchange,提问作者Shivika
相关产品推荐
相关产品推荐

