Python提取JSON指定键值对并将hit数据存入列表的实现问题

DevOps 小满

2026-9-26

问题描述

原始JSON文件每行是一个完整的JSON对象，包含访问基础信息、设备信息、地理位置信息和点击（hit）列表，需要逐行提取指定字段，最终输出每个访问对应的基础信息，以及该访问下所有hit组成的列表结构。

原始输入样例

{
    "visitNumber": "151",
    "visitId": "1470083489",
    "visitStartTime": "1470083489",
    "date": "20160801",
    "totals": {
        "visits": "1",
        "hits": "3",
        "pageviews": "3",
        "timeOnSite": "529"
    },
    "device": {
        "browser": "Chrome",
        "browserVersion": "not available in demo dataset",
        "browserSize": "not available in demo dataset",
        "operatingSystem": "Windows",
        "operatingSystemVersion": "not available in demo dataset",
        "isMobile": false,
        "mobileDeviceBranding": "not available in demo dataset",
        "mobileDeviceModel": "not available in demo dataset",
        "mobileInputSelector": "not available in demo dataset",
        "mobileDeviceInfo": "not available in demo dataset",
        "mobileDeviceMarketingName": "not available in demo dataset",
        "flashVersion": "not available in demo dataset",
        "language": "not available in demo dataset",
        "screenColors": "not available in demo dataset",
        "screenResolution": "not available in demo dataset",
        "deviceCategory": "desktop"
    },
    "geoNetwork": {
        "continent": "Americas",
        "subContinent": "Northern America",
        "country": "United States",
        "region": "California",
        "metro": "San Francisco-Oakland-San Jose CA",
        "city": "San Jose",
        "cityId": "not available in demo dataset",
        "networkDomain": "unknown.unknown",
        "latitude": "not available in demo dataset",
        "longitude": "not available in demo dataset",
        "networkLocation": "not available in demo dataset"
    },
    "hits": [
        {
            "hitNumber": "1",
            "time": "0",
            "hour": "13",
            "minute": "31",
            "isInteraction": true,
            "isEntrance": true,
            "page": {
                "pagePath": "/home",
                "hostname": "shop.googlemerchandisestore.com",
                "pageTitle": "Home",
                "pagePathLevel1": "/home",
                "pagePathLevel2": "",
                "pagePathLevel3": "",
                "pagePathLevel4": ""
            }
        },
        {
            "hitNumber": "2",
            "time": "289206",
            "hour": "13",
            "minute": "36",
            "isInteraction": true,
            "page": {
                "pagePath": "/google+redesign/apparel/men++s/men++s+t+shirts",
                "hostname": "shop.googlemerchandisestore.com",
                "pageTitle": "Men's-T-Shirts",
                "pagePathLevel1": "/google+redesign/",
                "pagePathLevel2": "/apparel/",
                "pagePathLevel3": "/men++s/",
                "pagePathLevel4": "/men++s+t+shirts"
            }
        }
       ]
  }

{
    "visitNumber": "1",
    "visitId": "1470117657",
    "visitStartTime": "1470117657",
    "date": "20160801",
    "totals": {
        "visits": "1",
        "hits": "3",
        "pageviews": "3",
        "timeOnSite": "54",
        "newVisits": "1"
    },
    "device": {
        "browser": "Internet Explorer",
        "browserVersion": "not available in demo dataset",
        "browserSize": "not available in demo dataset",
        "operatingSystem": "Windows",
        "operatingSystemVersion": "not available in demo dataset",
        "isMobile": false,
        "mobileDeviceBranding": "not available in demo dataset",
        "mobileDeviceModel": "not available in demo dataset",
        "mobileInputSelector": "not available in demo dataset",
        "mobileDeviceInfo": "not available in demo dataset",
        "mobileDeviceMarketingName": "not available in demo dataset",
        "flashVersion": "not available in demo dataset",
        "language": "not available in demo dataset",
        "screenColors": "not available in demo dataset",
        "screenResolution": "not available in demo dataset",
        "deviceCategory": "desktop"
    },
    "geoNetwork": {
        "continent": "Americas",
        "subContinent": "Northern America",
        "country": "United States",
        "region": "not available in demo dataset",
        "metro": "not available in demo dataset",
        "city": "not available in demo dataset",
        "cityId": "not available in demo dataset",
        "networkDomain": "telia.net",
        "latitude": "not available in demo dataset",
        "longitude": "not available in demo dataset",
        "networkLocation": "not available in demo dataset"
    },
    "hits": [
        {
            "hitNumber": "1",
            "time": "0",
            "hour": "23",
            "minute": "0",
            "isInteraction": true,
            "isEntrance": true,
            "referer": "https://www.youtube.com/yt/about/",
            "page": {
                "pagePath": "/home",
                "hostname": "shop.googlemerchandisestore.com",
                "pageTitle": "Home",
                "pagePathLevel1": "/home",
                "pagePathLevel2": "",
                "pagePathLevel3": "",
                "pagePathLevel4": ""
            }
         }
       ]
}

原有代码

import json
from datetime import datetime
from datetime import timedelta

visits_dict = {}
hits_dict = {}

hits_array = []

with open(r"C:\Users\rparpani\Documents\Test.json") as f:
    for line in f:
        json_object = json.loads(line)

        #This code builds the visit dictionary
        visits_dict["full_visitor_id"] = json_object["fullVisitorId"]
        visits_dict["visit_id"] = json_object["visitId"]
        visits_dict["visit_number"] = json_object["visitNumber"]

        visit_StartTime = datetime.fromtimestamp(int(json_object["visitStartTime"])).isoformat()
        visits_dict["visit_start_time"] = visit_StartTime

        visits_dict["browser"] = json_object["device"].get('browser')
        visits_dict["country"] = json_object["geoNetwork"].get('country')

        print(visits_dict)

        visitstartTime_unixValue = datetime.fromtimestamp(int(json_object["visitStartTime"]))
        #print(visitstartTime_unixValue)

        #This code outputs the data to a visits.json file
        analyticsInfo_visitsOutput = open("C:\\Users\\rparpani\\Documents\\visits.json", "a")
        json.dump(visits_dict, analyticsInfo_visitsOutput)
        analyticsInfo_visitsOutput.write('\n')

        # hits_arraycount = int(json_object.get('totals').get('hits'))
        # print(hits_arraycount)

        #This code builds the hits dictionary
        for j in json_object["hits"]:
            #print(j.get('hitNumber'))
            hits_dict["hit_number"] = j.get('hitNumber')
            hits_dict["hit_type"] = j.get('type')

            hit_timeValue = int(j.get('time'))
            time_ms = timedelta(milliseconds=hit_timeValue)
            new_time = visitstartTime_unixValue + time_ms
            hits_dict["hit_timestamp"] = new_time.isoformat()

            hits_dict["page_path"] = j.get('page').get('pagePath')
            hits_dict["page_title"] = j.get('page').get('pageTitle')
            hits_dict["hostname"] = j.get('page').get('hostname')   
            
            print(hits_dict)
            

        
        #This code outputs the data to a visits.json file
            analyticsInfo_hitsOutput = open("C:\\Users\\rparpani\\Documents\\hits.json", "a")
            json.dump(hits_dict, analyticsInfo_hitsOutput)
            analyticsInfo_hitsOutput.write('\n')

当前输出问题

当前代码会把每个hit单独作为一个JSON对象输出，没有按访问分组为列表结构：

{'full_visitor_id': '8159312408158297118', 'visit_id': '1470083489', 'visit_number': '151', 'visit_start_time': '2016-08-01T15:31:29', 'browser': 'Chrome', 'country': 'United States'}
{'hit_number': '1', 'hit_type': 'PAGE', 'hit_timestamp': '2016-08-01T15:31:29', 'page_path': '/home', 'page_title': 'Home', 'hostname': 'shop.googlemerchandisestore.com'}
{'hit_number': '2', 'hit_type': 'PAGE', 'hit_timestamp': '2016-08-01T15:36:18.206000', 'page_path': '/google+redesign/apparel/men++s/men++s+t+shirts', 'page_title': "Men's-T-Shirts", 'hostname': 'shop.googlemerchandisestore.com'}
{'hit_number': '3', 'hit_type': 'PAGE', 'hit_timestamp': '2016-08-01T15:40:17.730000', 'page_path': '/google+redesign/office', 'page_title': 'Office', 'hostname': 'shop.googlemerchandisestore.com'}

期望输出

{'full_visitor_id': '8159312408158297118', 'visit_id': '1470083489', 'visit_number': '151', 'visit_start_time': '2016-08-01T15:31:29', 'browser': 'Chrome', 'country': 'United States'}
{'hits':[{'hit_number': '1', 'hit_type': 'PAGE', 'hit_timestamp': '2016-08-01T15:31:29', 'page_path': '/home', 'page_title': 'Home', 'hostname': 'shop.googlemerchandisestore.com'}
{'hit_number': '2', 'hit_type': 'PAGE', 'hit_timestamp': '2016-08-01T15:36:18.206000', 'page_path': '/google+redesign/apparel/men++s/men++s+t+shirts', 'page_title': "Men's-T-Shirts", 'hostname': 'shop.googlemerchandisestore.com'}
{'hit_number': '3', 'hit_type': 'PAGE', 'hit_timestamp': '2016-08-01T15:40:17.730000', 'page_path': '/google+redesign/office', 'page_title': 'Office', 'hostname': 'shop.googlemerchandisestore.com'}]}

解决方案

修改要点

将visits_dict、hits_array的初始化移到每行JSON的处理逻辑内部，避免不同行数据互相覆盖
遍历hit时不要立刻写入文件，先新建单个hit字典，存入临时的hits数组中
所有hit遍历完成后，构造{"hits": hits数组}的结构，再统一写入文件
将文件打开逻辑移到循环外，避免频繁打开关闭文件降低效率

修改后的完整代码

import json
from datetime import datetime, timedelta

# 统一打开输入输出文件，避免频繁IO
with open(r"C:\Users\rparpani\Documents\Test.json", "r", encoding="utf-8") as f, \
     open(r"C:\Users\rparpani\Documents\visits.json", "a", encoding="utf-8") as visit_out, \
     open(r"C:\Users\rparpani\Documents\hits.json", "a", encoding="utf-8") as hit_out:
    
    for line in f:
        line = line.strip()
        if not line:
            continue  # 跳过空行
        json_object = json.loads(line)
        
        # 每次处理新行时初始化空容器，避免数据残留
        visits_dict = {}
        hits_array = []
        
        # 组装访问基础信息
        visits_dict["full_visitor_id"] = json_object["fullVisitorId"]
        visits_dict["visit_id"] = json_object["visitId"]
        visits_dict["visit_number"] = json_object["visitNumber"]
        visit_start_time = datetime.fromtimestamp(int(json_object["visitStartTime"])).isoformat()
        visits_dict["visit_start_time"] = visit_start_time
        visits_dict["browser"] = json_object["device"].get('browser')
        visits_dict["country"] = json_object["geoNetwork"].get('country')
        
        # 写入访问信息
        json.dump(visits_dict, visit_out, ensure_ascii=False)
        visit_out.write('\n')
        print(visits_dict)
        
        # 处理当前访问的所有hit
        visit_start_unix = datetime.fromtimestamp(int(json_object["visitStartTime"]))
        for j in json_object["hits"]:
            # 每个hit新建独立字典，避免覆盖
            hit_item = {}
            hit_item["hit_number"] = j.get('hitNumber')
            hit_item["hit_type"] = j.get('type')
            
            hit_time_ms = int(j.get('time'))
            time_delta = timedelta(milliseconds=hit_time_ms)
            hit_utc_time = visit_start_unix + time_delta
            hit_item["hit_timestamp"] = hit_utc_time.isoformat()
            
            hit_item["page_path"] = j.get('page').get('pagePath')
            hit_item["page_title"] = j.get('page').get('pageTitle')
            hit_item["hostname"] = j.get('page').get('hostname')   
            
            hits_array.append(hit_item)
            print(hit_item)
        
        # 组装hits列表结构，统一写入文件
        hit_output = {"hits": hits_array}
        json.dump(hit_output, hit_out, ensure_ascii=False)
        hit_out.write('\n')

内容的提问来源于stack exchange，提问作者GeekBoy

Python提取JSON指定键值对并将hit数据存入列表的实现问题

原始输入样例 #

原有代码 #

当前输出问题 #

期望输出 #

修改要点 #

修改后的完整代码 #

原始输入样例

原有代码

当前输出问题

期望输出

修改要点

修改后的完整代码