*- 也可能需要对 query 部分进行 URL 转码 percentencode 宗祠 local a = r(percentencode) percentencode 家庙 local b = r(percentencode) percentencode 祠堂 local c = r(percentencode)
*- 查看响应 insheetjson using "temp.json", showr flatten
*- 解析数据 clear gen str100 name = "" gen str100 lat = "" gen str100 lng = "" gen str100 address = "" gen str100 province = "" gen str100 city = "" gen str100 area = "" gen str100 street_id = "" gen str100 detail = "" gen str100 uid = "" insheetjson name lat lng address province city /// area street_id detail uid using "temp.json", /// table("results") col("name""location:lat"/// "location:lng""address""province""city"/// "area""street_id""detail""uid") compress destring, replace
这里仅仅处理了 page_num=0 页，如果结果超过 20（total）的话，就需要爬取多页了：
注意看接口文档中的 total 介绍：POI 检索总数，开发者请求中设置了 page_num 字段才会出现 total 字段。出于数据保护目的，单次请求 total 最多为150。
insheetjson name lat lng address /// province city area detail uid /// using temp2.json, table(results) /// col("name" "location:lat" "location:lng" "address" "province" /// "city" "area" "detail" "uid") offset(20)
insheetjson name lat lng address /// province city area detail uid /// using temp3.json, table(results) /// col("name" "location:lat" "location:lng" "address" "province" /// "city" "area" "detail" "uid") offset(20)
然后就可以循环爬取处理了。
对于复杂的网络数据爬取任务，建议先下载、下载完成后再集中解析。
首先循环下载首页 json 文件到工作目录下的 res 文件夹：
use"50km 网格中心.dta", clear
*- 循环每个观测值 capmkdir"res"
*- 爬取首页 forval i = 1/`=_N' { di"`i'" if !fileexists("res/`=id[`i']'_0.json") { copy"https://api.map.baidu.com/place/v2/search?query=宗祠$家庙$祠堂&location=`=Y[`i']',`=X[`i']'&radius=50000&output=json&ak=a8niO5h8icau3IRNG2Wv2k5CduVyrktL&coord_tyle=wgs84ll&ret_coordtype=gcj02ll&page_size=20&page_num=0""res/`=id[`i']'_0.json", replace } }
从这些首页 json 文件里面提取 total 值：
*- 提取 total 结果 clear all gen str10 total = "" insheetjson total using "temp.json", table("total") offset(0)
use"50km 网格中心.dta", clear tostring id, replace genfile = "res/" + id + "_0.json" gen str10 total = "" forval i = 1/`=_N' { insheetjson total using "`=file[`i']'", table("total") offset(`=`i' - 1') }
destring, replace
*- total <= 20 的都是只有一页的 dropiftotal <= 20
gen pagenum = int(total/20)
*- 循环下载其余页面 forval i = 1/`=_N' { forval j = 1/`=pagenum[`i']' { if !fileexists("res/`=id[`i']'_`j'.json") { copy"https://api.map.baidu.com/place/v2/search?query=宗祠$家庙$祠堂&location=`=Y[`i']',`=X[`i']'&radius=50000&output=json&ak=a8niO5h8icau3IRNG2Wv2k5CduVyrktL&coord_tyle=wgs84ll&ret_coordtype=gcj02ll&page_size=20&page_num=`j'""res/`=id[`i']'_`j'.json", replace } } }
循环处理下载得到的文件：
local files: dir"res" files "*.json" di`"`files'"'
clear gen str100 name = "" gen str100 lat = "" gen str100 lng = "" gen str100 address = "" gen str100 province = "" gen str100 city = "" gen str100 area = "" gen str100 detail = "" gen str100 uid = ""
foreach i in`files' { localN = `=_N' di"`i'" insheetjson name lat lng address province city /// area detail uid using "res/`i'", /// table("results") col("name""location:lat"/// "location:lng""address""province""city"/// "area""detail""uid") offset(`N') }
import excel using "全国宗庙祠堂分布（10km 范围的检索）.xlsx", clear first geoinpoly lat lng using county_coord.dta ren _ID ID mergem:1 ID using county_db dropifmissing(lng) drop 省 省代码 市 县 县代码 县类型 市类型 省类型 _merge gen id = _n *- 计算每个城市的宗庙祠堂密度 collapse (count) count = id, by(市代码)
*- 匹配面积数据 merge 1:1 市代码 using 2020年各城市行政区划面积.dta replacecount = 0 ifmissing(count) keepcount 市 市代码 面积 *- 面积的单位是平方公里 gen density = count / (面积 / 10000) labelvar density "宗庙祠堂密度（个/万平方公里）" save"各城市宗庙祠堂密度", replace
*- 绘图 use chinacity2020mini_db.dta, clear merge 1:1 市 市代码 using "各城市宗庙祠堂密度" keepif _m == 3 replace density = 0 ifmissing(density)
评论