使用Node.js抓取Google Maps隐藏API：解析返回文本文件问题

问题描述 #

我正在用Node.js通过以下URL抓取Google Maps搜索结果：

https://www.google.com/search?q=pizza&hl=en&tbm=map&tch=1&pb=!4m8!1m3!1d11281.305980319747!2d-74.0083012!3d40.7455096!3m2!1i1024!2i768!4f13.1!7i20!8i40!10b1!12m25!1m1!18b1!2m3!5m1!6e2!20e3!6m16!4b1!23b1!26i1!27i1!41i2!45b1!49b1!63m0!67b1!73m0!74i150000!75b1!89b1!105b1!109b1!110m0!10b1!16b1!19m4!2m3!1i360!2i120!4i8!20m65!2m2!1i203!2i100!3m2!2i4!5b1!6m6!1m2!1i86!2i86!1m2!1i408!2i240!7m50!1m3!1e1!2b0!3e3!1m3!1e2!2b1!3e2!1m3!1e2!2b0!3e3!1m3!1e3!2b0!3e3!1m3!1e8!2b0!3e3!1m3!1e3!2b1!3e2!1m3!1e10!2b0!3e3!1m3!1e10!2b1!3e2!1m3!1e9!2b1!3e2!1m3!1e10!2b0!3e3!1m3!1e10!2b1!3e2!1m3!1e10!2b0!3e4!2b1!4b1!9b0!22m3!1s!2z!7e81!24m55!1m15!13m7!2b1!3b1!4b1!6i1!8b1!9b1!20b0!18m6!3b1!4b1!5b1!6b1!13b0!14b0!2b1!5m5!2b1!3b1!5b1!6b1!7b1!10m1!8e3!14m1!3b1!17b1!20m4!1e3!1e6!1e14!1e15!24b1!25b1!26b1!29b1!30m1!2b1!36b1!43b1!52b1!54m1!1b1!55b1!56m2!1b1!3b1!65m5!3m4!1m3!1m2!1i224!2i298!89b1!26m4!2m3!1i80!2i92!4i8!30m28!1m6!1m2!1i0!2i0!2m2!1i458!2i768!1m6!1m2!1i974!2i0!2m2!1i1024!2i768!1m6!1m2!1i0!2i0!2m2!1i1024!2i20!1m6!1m2!1i0!2i748!2m2!1i1024!2i768!34m16!2b1!3b1!4b1!6b1!8m4!1b1!3b1!4b1!6b1!9b1!12b1!14b1!20b1!23b1!25b1!26b1!37m1!1e81!42b1!46m1!1e9!47m0!49m1!3b1!50m53!1m49!2m7!1u3!4s!5e1!9s!10m2!3m1!1e1!2m7!1u2!4s!5e1!9s!10m2!2m1!1e1!2m7!1u16!4s!5e1!9s!10m2!16m1!1e1!2m7!1u16!4s!5e1!9s!10m2!16m1!1e2!3m11!1u16!2m4!1m2!16m1!1e1!2s!2m4!1m2!16m1!1e2!2s!3m1!1u2!3m1!1u3!4BIAE!2e2!3m1!3b1!59B!65m0!69i540

在浏览器打开这个URL会下载一个文本文件，但我识别不出它的格式（是否为JSON），也不知道怎么解析里面的数据。以下是我的实现代码：

const cheerio = require("cheerio");
const fs = require("fs");
const unirest = require("unirest");

const getData = async () => {
  try {
    const url =
      "https://www.google.com/search?q=pizza&hl=en&tbm=map&tch=1&pb=!4m8!1m3!1d11281.305980319747!2d-74.0083012!3d40.7455096!3m2!1i1024!2i768!4f13.1!7i20!8i40!10b1!12m25!1m1!18b1!2m3!5m1!6e2!20e3!6m16!4b1!23b1!26i1!27i1!41i2!45b1!49b1!63m0!67b1!73m0!74i150000!75b1!89b1!105b1!109b1!110m0!10b1!16b1!19m4!2m3!1i360!2i120!4i8!20m65!2m2!1i203!2i100!3m2!2i4!5b1!6m6!1m2!1i86!2i86!1m2!1i408!2i240!7m50!1m3!1e1!2b0!3e3!1m3!1e2!2b1!3e2!1m3!1e2!2b0!3e3!1m3!1e3!2b0!3e3!1m3!1e8!2b0!3e3!1m3!1e3!2b1!3e2!1m3!1e10!2b0!3e3!1m3!1e10!2b1!3e2!1m3!1e9!2b1!3e2!1m3!1e10!2b0!3e3!1m3!1e10!2b1!3e2!1m3!1e10!2b0!3e4!2b1!4b1!9b0!22m3!1s!2z!7e81!24m55!1m15!13m7!2b1!3b1!4b1!6i1!8b1!9b1!20b0!18m6!3b1!4b1!5b1!6b1!13b0!14b0!2b1!5m5!2b1!3b1!5b1!6b1!7b1!10m1!8e3!14m1!3b1!17b1!20m4!1e3!1e6!1e14!1e15!24b1!25b1!26b1!29b1!30m1!2b1!36b1!43b1!52b1!54m1!1b1!55b1!56m2!1b1!3b1!65m5!3m4!1m3!1m2!1i224!2i298!89b1!26m4!2m3!1i80!2i92!4i8!30m28!1m6!1m2!1i0!2i0!2m2!1i458!2i768!1m6!1m2!1i974!2i0!2m2!1i1024!2i768!1m6!1m2!1i0!2i0!2m2!1i1024!2i20!1m6!1m2!1i0!2i748!2m2!1i1024!2i768!34m16!2b1!3b1!4b1!6b1!8m4!1b1!3b1!4b1!6b1!9b1!12b1!14b1!20b1!23b1!25b1!26b1!37m1!1e81!42b1!46m1!1e9!47m0!49m1!3b1!50m53!1m49!2m7!1u3!4s!5e1!9s!10m2!3m1!1e1!2m7!1u2!4s!5e1!9s!10m2!2m1!1e1!2m7!1u16!4s!5e1!9s!10m2!16m1!1e1!2m7!1u16!4s!5e1!9s!10m2!16m1!1e2!3m11!1u16!2m4!1m2!16m1!1e1!2s!2m4!1m2!16m1!1e2!2s!3m1!1u2!3m1!1u3!4BIAE!2e2!3m1!3b1!59B!65m0!69i540";

    const response = await unirest.get(url).headers({
      "User-Agent":
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36",
    });

    const $ = cheerio.load(response.body);
    fs.writeFileSync("./maps.txt", response.body);
  }
  catch (e) {
    console.log(e);
  }
};

getData();

解析方案 #

1. 识别文件格式 #

这个下载的文件是Google Protocol Buffers（Protobuf）格式，并非JSON。它是Google用于高效数据传输的二进制序列化格式，你看到的文本是二进制数据转码后的ASCII内容。

2. 解析步骤 #

要解析这类数据，需要以下操作：

• 获取对应Google Maps搜索结果的Protobuf定义文件（.proto）
• 使用JavaScript的Protobuf库（如protobufjs）编译定义并解析二进制数据

具体实现代码

首先安装依赖：

npm install protobufjs

修改原代码，添加解析逻辑：

const fs = require("fs");
const unirest = require("unirest");
const protobuf = require("protobufjs");

const getData = async () => {
  try {
    const url = "你的目标URL"; // 保留原URL不变

    const response = await unirest.get(url).headers({
      "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.54 Safari/537.36",
    });

    // 将响应文本转换为二进制Buffer
    const buffer = Buffer.from(response.body, "binary");

    // 加载Google Maps搜索结果的Protobuf定义文件
    const root = await protobuf.load("maps_search.proto");
    const SearchResult = root.lookupType("google.maps.SearchResult");

    // 解析二进制数据
    const message = SearchResult.decode(buffer);
    const parsedResult = SearchResult.toObject(message, {
      enums: String,
      longs: String,
      bytes: String,
      defaults: true,
      arrays: true,
      objects: true,
      oneofs: true
    });

    // 将解析结果写入JSON文件
    fs.writeFileSync("./maps_result.json", JSON.stringify(parsedResult, null, 2));
    console.log("解析完成，结果已保存到maps_result.json");
  } catch (e) {
    console.log(e);
  }
};

getData();

3. 获取Protobuf定义文件 #

Google未公开Maps搜索结果的官方Protobuf定义，可通过以下方式获取：

• 抓包浏览器请求，分析Protobuf数据结构
• 在代码托管平台搜索公开的Google Maps Protobuf定义（注意合规性）

注意事项 #

• URL中的pb参数是动态生成的，Google反爬机制更新后可能失效
• 未经授权抓取Google Maps数据可能违反其服务条款，建议使用官方的Google Maps Places API获取数据

内容的提问来源于stack exchange，提问作者Jimmy