如何将多级导航菜单数据动态输出到CSV多列

问题 #

我是Python新手，正在爬取网站导航菜单并尝试将数据存储到CSV文件中。脚本能正常运行并存储数据，但所有数据都保存到了CSV的第一列，无法分存到多列。我希望将菜单的不同层级分别存入CSV的对应列（如level 1到level 5列），曾尝试硬编码指定列内容但无效，现寻求动态实现该需求的方法。

当前代码如下：

import csv, json, sys
import bs4, requests


def to_dict(r):
   if all(isinstance(i, dict) for i in r):
      return {a:b for i in r for a, b in i.items()}
   if all(isinstance(i, list) for i in r):
      return [i for b in r for i in b]
   return r

def get_menu(page_name):
   if page_name.name == 'li':
      #collect all li values, create a dictionary if li has a ul child
      li_text = []
      for i in page_name.contents:
         if isinstance(i, bs4.element.NavigableString) or i.select_one('ul') is None:
           if (t:=i.get_text(strip=True)): #default li text
              li_text.append(t)
         else: #li contains a ul, build a dict with li's text as key
            if li_text:
               li_text.append({li_text.pop():to_dict([*get_menu(i)])})
               
            else:
               yield to_dict([*get_menu(i)])
      yield from li_text
      return
   all_text = []
   for i in page_name.contents:
      if getattr(i, 'name', None) == 'ul':
         r = [*get_menu(i)] #get ul's li values
         e, flag = i, False
         while (e:=e.previous_sibling) is not None: #check if ul has descriptive text
           if (isinstance(e, bs4.element.NavigableString) or e.select_one('ul') is None) and e.get_text(strip=True):
              yield {e.get_text(strip=True):to_dict(r)} #yield back dictionary if text exists
              flag = True
              break
         if not flag: yield to_dict(r)
      elif not isinstance(i, bs4.element.NavigableString):
         yield from get_menu(i) #recursively traverse element

def menu_items(link):
   page = bs4.BeautifulSoup(requests.get(link).text, 'html.parser')
   for nav in page.select('nav'): #find all `nav` elements from which to anchor search
      yield to_dict([*get_menu(nav)])
     


for i in menu_items('https://www.turramusic.com.au/'):
   
   f = csv.writer(open(r"C:\Users\Manikanta-WD\Desktop\run\csv\url_output.csv", "w"))
   f.writerow(["level 1", "level 2", "level 3", "level 4", "level 5"])
   for x in i:
    f.writerow([x])

曾尝试以下硬编码方式，但无法正常运行且非动态实现：

for x in x:
    f.writerow([x["Keyboards & Synths"],
                x["Drums & Percussion"],
                x["fields"]["Band & Orchestral'"],
                x["fields"]["Live Sound"],
                x["fields"]["Amps & Effects"]])

需求：动态实现菜单层级对应CSV列（第一层级存第一列，第二层级存第二列等）。

解决方案 #

核心问题是当前代码输出的是嵌套字典/列表结构，需要先把嵌套菜单转换成扁平的层级路径列表，再写入CSV。

1. 添加递归扁平化函数 #

新增一个函数，把嵌套的菜单结构转换成每个层级对应的列表项，比如["乐器", "键盘合成器", "合成器"]这样的格式：

def flatten_menu(menu, current_level=None):
    # 初始化当前层级路径
    if current_level is None:
        current_level = []
    items = []
    if isinstance(menu, dict):
        for key, value in menu.items():
            # 把当前键加入层级路径
            new_level = current_level + [key]
            items.append(new_level)
            # 递归处理子菜单
            items.extend(flatten_menu(value, new_level))
    elif isinstance(menu, list):
        for item in menu:
            if isinstance(item, (dict, list)):
                items.extend(flatten_menu(item, current_level))
            else:
                # 处理纯文本菜单项
                items.append(current_level + [item])
    return items

2. 修改CSV写入逻辑 #

原来的写入逻辑只把单个元素放到第一列，现在需要把扁平化后的层级列表填充到对应列，不足的层级留空：

# 处理菜单数据
for menu in menu_items('https://www.turramusic.com.au/'):
    # 扁平化菜单
    flat_menu = flatten_menu(menu)
    # 确定最大层级数（动态生成列标题）
    max_levels = max(len(item) for item in flat_menu) if flat_menu else 5
    headers = [f"level {i+1}" for i in range(max_levels)]
    
    # 写入CSV
    with open(r"C:\Users\Manikanta-WD\Desktop\run\csv\url_output.csv", "w", newline="", encoding="utf-8") as f:
        writer = csv.writer(f)
        writer.writerow(headers)
        for item in flat_menu:
            # 填充空值到最大层级数
            row = item + [""] * (max_levels - len(item))
            writer.writerow(row)

3. 完整修改后的代码 #

把所有部分整合起来：

import csv
import bs4
import requests

def to_dict(r):
    if all(isinstance(i, dict) for i in r):
        return {a: b for i in r for a, b in i.items()}
    if all(isinstance(i, list) for i in r):
        return [i for b in r for i in b]
    return r

def get_menu(page_name):
    if page_name.name == 'li':
        li_text = []
        for i in page_name.contents:
            if isinstance(i, bs4.element.NavigableString) or i.select_one('ul') is None:
                if (t := i.get_text(strip=True)):
                    li_text.append(t)
            else:
                if li_text:
                    li_text.append({li_text.pop(): to_dict([*get_menu(i)])})
                else:
                    yield to_dict([*get_menu(i)])
        yield from li_text
        return
    for i in page_name.contents:
        if getattr(i, 'name', None) == 'ul':
            r = [*get_menu(i)]
            e, flag = i, False
            while (e := e.previous_sibling) is not None:
                if (isinstance(e, bs4.element.NavigableString) or e.select_one('ul') is None) and e.get_text(strip=True):
                    yield {e.get_text(strip=True): to_dict(r)}
                    flag = True
                    break
            if not flag:
                yield to_dict(r)
        elif not isinstance(i, bs4.element.NavigableString):
            yield from get_menu(i)

def menu_items(link):
    page = bs4.BeautifulSoup(requests.get(link).text, 'html.parser')
    for nav in page.select('nav'):
        yield to_dict([*get_menu(nav)])

def flatten_menu(menu, current_level=None):
    if current_level is None:
        current_level = []
    items = []
    if isinstance(menu, dict):
        for key, value in menu.items():
            new_level = current_level + [key]
            items.append(new_level)
            items.extend(flatten_menu(value, new_level))
    elif isinstance(menu, list):
        for item in menu:
            if isinstance(item, (dict, list)):
                items.extend(flatten_menu(item, current_level))
            else:
                items.append(current_level + [item])
    return items

# 主逻辑
for menu in menu_items('https://www.turramusic.com.au/'):
    flat_menu = flatten_menu(menu)
    if not flat_menu:
        continue
    max_levels = max(len(item) for item in flat_menu)
    headers = [f"level {i+1}" for i in range(max_levels)]
    
    with open(r"C:\Users\Manikanta-WD\Desktop\run\csv\url_output.csv", "w", newline="", encoding="utf-8") as f:
        writer = csv.writer(f)
        writer.writerow(headers)
        for item in flat_menu:
            row = item + [""] * (max_levels - len(item))
            writer.writerow(row)

说明 #

• flatten_menu函数递归遍历嵌套的菜单结构，把每个层级的路径转换成列表，比如一级菜单+二级菜单的组合会变成[level1, level2]，三级菜单变成[level1, level2, level3]。
• CSV写入时会根据最大层级数动态生成列标题，不足的层级用空字符串填充，确保每一行的列数一致。
• 使用with open管理文件，避免资源泄漏，同时指定encoding="utf-8"防止字符乱码。

内容的提问来源于stack exchange，提问作者manikanta k j gowda