Wigle API按经纬度获取全量数据遇分页限制，如何解决？

问题详情 #

我用Go编写脚本调用Wigle API v2的/api/v2/network/search端点，想要按指定经纬度范围获取全量Wi-Fi数据，但目前最多只能获取1000个唯一SSID。当前尝试通过迭代更新URL中的searchAfter参数（搭配first参数）实现分页，但单独使用searchAfter只能拿到100条数据，单独用first参数最多拿到1000条，两种方式组合也无法获取全量结果。

请求URL示例 #

• 首次请求：

  https://api.wigle.net/api/v2/network/search?closestLat=12.9&closestLong=1.2&latrange1=1.9&latrange2=1.8&longrange1=1.2&longrange2=1.4
  

• 后续分页请求：

  https://api.wigle.net/api/v2/network/search?closestLat=12.9&closestLong=1.2&latrange1=1.9&latrange2=1.8&longrange1=1.2&longrange2=1.4&searchAfter=1976621348&first=1
  

我的Go代码实现 #

var (
    wg          = sync.WaitGroup{}
    receiveResp = make(chan []*response.WiFiNetworkWithLocation, 100)
)

func main() {
    startTime := time.Now()

    viper.AddConfigPath(".")
    viper.SetConfigFile("config.json")
    if err := viper.ReadInConfig(); err != nil {
        log.Fatal("error trying read from config: %w", err)
    }

    u := user.NewUser(viper.GetString("users.user.username"), viper.GetString("users.user.password"))

    db, err := postgres.NewPG()
    if err != nil {
        log.Fatalf("Cannot create postgres connection: %v", err)
    }

    postgres.WG.Add(1)
    go getResponse(u)

    go parseResponse(db)

    postgres.WG.Wait()

    fmt.Printf("Execution time: %v ", time.Since(startTime))
}

func getResponse(u *user.Creds) {
    url := fmt.Sprintf("%s?closestLat=%s&closestLong=%s&latrange1=%s&latrange2=%s&longrange1=%s&longrange2=%s",
        viper.GetString("wigle.url"),
        viper.GetString("queries.closestLat"),
        viper.GetString("queries.closestLong"),
        viper.GetString("queries.latrange1"),
        viper.GetString("queries.latrange2"),
        viper.GetString("queries.longrange1"),
        viper.GetString("queries.longrange2"),
    )

    j := 0
    i := 0
    for {
        i++

        fmt.Println(url)

        req, err := http.NewRequest("GET", url, bytes.NewBuffer([]byte("")))
        if err != nil {
            log.Printf("Failed wraps request: %v", err)
            continue
        }
        req.SetBasicAuth(u.Username, u.Password)

        c := http.Client{}
        resp, err := c.Do(req)
        if err != nil {
            log.Printf("Failed send request: %v", err)
            continue
        }

        bytes, err := ioutil.ReadAll(resp.Body)
        if err != nil {
            log.Printf("Failed read response body: %v", err)
            continue
        }

        var r response.NetSearchResponse

        if err := json.Unmarshal(bytes, &r); err != nil {
            log.Printf("Failed unmarshal: %v", err)
            continue
        }

        receiveResp <- r.Results

        fmt.Println(r.TotalResults, r.SearchAfter)

        if r.SearchAfter == "" {
            postgres.WG.Done()
            return
        }

        url = fmt.Sprintf("%s?closestLat=%s&closestLong=%s&latrange1=%s&latrange2=%s&longrange1=%s&longrange2=%s&searchAfter=%s&first=%v",
            viper.GetString("wigle.url"),
            viper.GetString("queries.closestLat"),
            viper.GetString("queries.closestLong"),
            viper.GetString("queries.latrange1"),
            viper.GetString("queries.latrange2"),
            viper.GetString("queries.longrange1"),
            viper.GetString("queries.longrange2"),
            r.SearchAfter,
            i,
        )
        j++

        fmt.Println(j)
    }
}

func parseResponse(db *sql.DB) {
    for {
        select {
        case responses := <-receiveResp:
            clearResponses := make([]response.WiFiNetworkWithLocation, 0, len(responses))

            for _, val := range responses {
                clearResponses = append(clearResponses, *val)
            }
            postgres.WG.Add(1)
            go postgres.SaveToDB(db, "test", clearResponses)
        }
    }
}

内容的提问来源于stack exchange，提问作者alemax1