上市公司年报的获取方法很多，今天我们一起学习下如何从巨潮资讯网爬取，非常好的一点是，巨潮资讯网并不反爬，所以爬取起来简单很多。
爬取的网站网址是：http://www.cninfo.com.cn/new/commonUrl/pageOfSearch?url=disclosure/list/search
从网站右侧的筛选窗口可以筛选报告下载。所以我的爬取思路是，首先获取全部可选的上市公司列表，然后逐个公司爬取其历年年报的链接然后再下载。
获取所有可选的上市公司列表
通过网页分析，可以找到可选的上市公司列表数据在这里：
是一个 json 格式的文件，可以使用 jsonlite 处理：
library(tidyverse)
library(jsonlite)
fromJSON("http://www.cninfo.com.cn/new/data/szse_stock.json") -> lst
lst$stockList %>%
as_tibble() %>%
mutate(stext = paste0(code, ",", orgId)) -> codelist
codelist
|
一共是 5622 家上市公司。
爬取单家公司的年报链接
以平安银行近三年的年报为例：
对着 query 右键复制 curl 代码：
curl 'http://www.cninfo.com.cn/new/hisAnnouncement/query' \
-H 'Accept: */*' \
-H 'Accept-Language: zh-CN,zh;q=0.9,en;q=0.8' \
-H 'Connection: keep-alive' \
-H 'Content-Type: application/x-www-form-urlencoded; charset=UTF-8' \
-H 'Cookie: JSESSIONID=CEC404925A4CD863DB2CD110942E1766; insert_cookie=45380249; _sp_ses.2141=*; routeId=.uc2; _sp_id.2141=b13ee9ff-2493-4f42-a7f5-089a2990e863.1694410049.9.1694788568.1694748531.2d165bfc-a649-4d35-9e23-4e3b6304e9d5' \
-H 'Origin: http://www.cninfo.com.cn' \
-H 'Referer: http://www.cninfo.com.cn/new/commonUrl/pageOfSearch?url=disclosure/list/search' \
-H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36' \
-H 'X-Requested-With: XMLHttpRequest' \
--data-raw 'pageNum=1&pageSize=30&column=szse&tabName=fulltext&plate=&stock=000001%2Cgssz0000001&searchkey=&secid=&category=category_ndbg_szsh&trade=&seDate=2020-09-15~2023-09-16&sortName=&sortType=&isHLtitle=true' \
--compressed \
--insecure
|
curl 语句是在 DOS 窗口或者终端里面运行的，可以使用这个网页应用把 curl 代码转换成 R 语言代码：https://curlconverter.com/r/
require(httr)
cookies = c(
`JSESSIONID` = "CEC404925A4CD863DB2CD110942E1766",
`insert_cookie` = "45380249",
`_sp_ses.2141` = "*",
`routeId` = ".uc2",
`_sp_id.2141` = "b13ee9ff-2493-4f42-a7f5-089a2990e863.1694410049.9.1694788568.1694748531.2d165bfc-a649-4d35-9e23-4e3b6304e9d5"
)
headers = c(
`Accept` = "*/*",
`Accept-Language` = "zh-CN,zh;q=0.9,en;q=0.8",
`Connection` = "keep-alive",
`Content-Type` = "application/x-www-form-urlencoded; charset=UTF-8",
`Origin` = "http://www.cninfo.com.cn",
`Referer` = "http://www.cninfo.com.cn/new/commonUrl/pageOfSearch?url=disclosure/list/search",
`User-Agent` = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36",
`X-Requested-With` = "XMLHttpRequest"
)
data = list(
`pageNum` = "1",
`pageSize` = "30",
`column` = "szse",
`tabName` = "fulltext",
`plate` = "",
`stock` = "000001,gssz0000001",
`searchkey` = "",
`secid` = "",
`category` = "category_ndbg_szsh",
`trade` = "",
`seDate` = "2020-09-15~2023-09-16",
`sortName` = "",
`sortType` = "",
`isHLtitle` = "true"
)
res <- httr::POST(url = "http://www.cninfo.com.cn/new/hisAnnouncement/query", httr::add_headers(.headers=headers), httr::set_cookies(.cookies = cookies), body = data, encode = "form", config = httr::config(ssl_verifypeer = FALSE))
|
我们可以试试把 seDate 修改成 “2000-01-01~2023-09-16”（pageSize 也可以试试修改成 100，不过没什么效果）：
data = list(
`pageNum` = "1",
`pageSize` = "30",
`column` = "szse",
`tabName` = "fulltext",
`plate` = "",
`stock` = "000001,gssz0000001",
`searchkey` = "",
`secid` = "",
`category` = "category_ndbg_szsh",
`trade` = "",
`seDate` = "2000-01-01~2023-09-16",
`sortName` = "",
`sortType` = "",
`isHLtitle` = "true"
)
res <- httr::POST(url = "http://www.cninfo.com.cn/new/hisAnnouncement/query", httr::add_headers(.headers=headers), httr::set_cookies(.cookies = cookies), body = data, encode = "form", config = httr::config(ssl_verifypeer = FALSE))
content(res) -> ls
ls$announcements %>%
transpose() %>%
as_tibble() %>%
unnest() %>%
filter(!str_detect(announcementTitle, "摘要")) %>%
select(announcementTitle, adjunctUrl) %>%
mutate(adjunctUrl = paste0("http://static.cninfo.com.cn/", adjunctUrl)) -> df1
df1
|
这样就得到了第一页的结果，类似的方法再获取第二页的：
data = list(
`pageNum` = "2",
`pageSize` = "30",
`column` = "szse",
`tabName` = "fulltext",
`plate` = "",
`stock` = "000001,gssz0000001",
`searchkey` = "",
`secid` = "",
`category` = "category_ndbg_szsh",
`trade` = "",
`seDate` = "2000-01-01~2023-09-16",
`sortName` = "",
`sortType` = "",
`isHLtitle` = "true"
)
res <- httr::POST(url = "http://www.cninfo.com.cn/new/hisAnnouncement/query", httr::add_headers(.headers=headers), httr::set_cookies(.cookies = cookies), body = data, encode = "form", config = httr::config(ssl_verifypeer = FALSE))
content(res) -> ls
ls$announcements %>%
transpose() %>%
as_tibble() %>%
unnest() %>%
filter(!str_detect(announcementTitle, "摘要")) %>%
select(announcementTitle, adjunctUrl) %>%
mutate(adjunctUrl = paste0("http://static.cninfo.com.cn/", adjunctUrl)) -> df2
df2
|
不过可能会有多页，所以还是循环比较好：
lapply(1:3, function(x){
data = list(
`pageNum` = x,
`pageSize` = "30",
`column` = "szse",
`tabName` = "fulltext",
`plate` = "",
`stock` = "000001,gssz0000001",
`searchkey` = "",
`secid` = "",
`category` = "category_ndbg_szsh",
`trade` = "",
`seDate` = "2000-01-01~2023-09-11",
`sortName` = "",
`sortType` = "",
`isHLtitle` = "true"
)
httr::POST(url = "http://www.cninfo.com.cn/new/hisAnnouncement/query",
httr::add_headers(.headers=headers),
httr::set_cookies(.cookies = cookies),
body = data, encode = "form",
config = httr::config(ssl_verifypeer = FALSE)) %>%
content() -> ls
if (length(ls$announcements) > 0) {
ls$announcements %>%
transpose() %>%
as_tibble() %>%
unnest() %>%
filter(!str_detect(announcementTitle, "摘要")) %>%
select(announcementTitle, adjunctUrl) %>%
mutate(adjunctUrl = paste0("http://static.cninfo.com.cn/", adjunctUrl),
code = "000001")
}
}) %>%
bind_rows() -> dfall
dfall
|
这样我们就得到了一家公司所有年份的年报文件链接了。
爬取全部上市公司的年报链接
循环所有的上市公司代码即可，这里仅展示前 6 家公司的：
dir.create("rds")
lapply(1:6, function(i){
print(codelist$zwjc[i])
lapply(1:5, function(x){
data = list(
`pageNum` = x,
`pageSize` = "30",
`column` = "szse",
`tabName` = "fulltext",
`plate` = "",
`stock` = codelist$stext[i],
`searchkey` = "",
`secid` = "",
`category` = "category_ndbg_szsh",
`trade` = "",
`seDate` = "1990-01-01~2023-09-16",
`sortName` = "",
`sortType` = "",
`isHLtitle` = "true"
)
httr::POST(url = "http://www.cninfo.com.cn/new/hisAnnouncement/query",
httr::add_headers(.headers=headers),
httr::set_cookies(.cookies = cookies),
body = data, encode = "form",
config = httr::config(ssl_verifypeer = FALSE)) %>%
content() -> ls
if (length(ls$announcements) > 0) {
ls$announcements %>%
transpose() %>%
as_tibble() %>%
unnest() %>%
filter(!str_detect(announcementTitle, "摘要")) %>%
select(announcementTitle, adjunctUrl) %>%
mutate(adjunctUrl = paste0("http://static.cninfo.com.cn/", adjunctUrl),
code = codelist$code[i]) -> tempdf
tempdf %>%
write_rds(paste0("rds/", i, "_", x, ".rds"))
return(tempdf)
}
}) %>%
bind_rows()
}) %>%
bind_rows() -> dfall
|
这里为了避免爬取过程中出现问题，我把每家公司爬取的结果也分别保存到了 rds 文件夹中，方便再次循环的时候跳过这些已经爬取好的。再次循环可以这样：
dir.create("rds")
lapply(1:6, function(i){
print(codelist$zwjc[i])
lapply(1:5, function(x){
if(!file.exists(paste0("rds/", i, "_", x, ".rds"))) {
data = list(
`pageNum` = x,
`pageSize` = "30",
`column` = "szse",
`tabName` = "fulltext",
`plate` = "",
`stock` = codelist$stext[i],
`searchkey` = "",
`secid` = "",
`category` = "category_ndbg_szsh",
`trade` = "",
`seDate` = "1990-01-01~2023-09-16",
`sortName` = "",
`sortType` = "",
`isHLtitle` = "true"
)
httr::POST(url = "http://www.cninfo.com.cn/new/hisAnnouncement/query",
httr::add_headers(.headers=headers),
httr::set_cookies(.cookies = cookies),
body = data, encode = "form",
config = httr::config(ssl_verifypeer = FALSE)) %>%
content() -> ls
if (length(ls$announcements) > 0) {
ls$announcements %>%
transpose() %>%
as_tibble() %>%
unnest() %>%
filter(!str_detect(announcementTitle, "摘要")) %>%
select(announcementTitle, adjunctUrl) %>%
mutate(adjunctUrl = paste0("http://static.cninfo.com.cn/", adjunctUrl),
code = codelist$code[i]) -> tempdf
tempdf %>%
write_rds(paste0("rds/", i, "_", x, ".rds"))
return(tempdf)
}
}
}) %>%
bind_rows()
}) -> res
fs::dir_ls("rds") %>%
lapply(readr::read_rds) %>%
bind_rows() -> dfall
dfall
|
这样会更加稳健。
爬取到的结果中还有很多是补充报告或者英文报告等，这些文件如果不需要的话可以筛选出来删除：
dfall %>%
filter(str_detect(announcementTitle, "英文|补充"))
dfall %>%
filter(!str_detect(announcementTitle, "英文|补充")) %>%
rename(title = announcementTitle, url = adjunctUrl) %>%
mutate(year = str_extract(title, "\\d{4}")) %>%
mutate(title = paste0(code, "_", year, "_", title)) -> dfall
dfall
|
另外，里面还有一些同一年两份文件的情况，一份是更新前的，一份是更新后的。这种由于命名方式多种多样，所以得谨慎删除。不过这些更新的内容往往都很少，使用更新前的还是更新后的实际上对文本分析的结果几乎没有影响，所以也不用太在意。
批量下载 pdf 文件
如果有时间的话，可以更加认真的筛选要下载的 pdf 文件。筛选好之后就可以下载了，也很简单：
dir.create("pdf")
lapply(1:nrow(dfall), function(x){
if(!file.exists(paste0("pdf/", dfall$title[x], ".pdf"))) {
print(x)
try({
download.file(dfall$url[x], paste0("pdf/", dfall$title[x], ".pdf"), mode = "wb")
})
}
}) -> res
|
如果要下载的 pdf 文件很多，可以考虑使用多线程下载，效率很快很多。
library(parallel)
detectCores()
makeCluster(36) -> cl
clusterExport(cl, "dfall")
parLapply(cl, 1:nrow(dfall), function(x){
if(!file.exists(paste0("pdf/", dfall$title[x], ".pdf"))) {
print(x)
try({
download.file(dfall$url[x], paste0("pdf/", dfall$title[x], ".pdf"), mode = "wb")
})
}
}) -> res
|
下载完之后可能还需要手动检查有没有下载失败的再重新下载。
文件管理
完整下载所有的年报后会得到巨多的 pdf 文件，难以管理，这里我们可以使用代码索引文件筛选自己需要的：
fs::dir_ls("pdf") %>%
as.character() %>%
as_tibble() -> fl
fl %>%
tidyr::extract(col = value, into = c("code", "year"),
regex = "/(.*)_(\\d{4})_", remove = F)
|
点击这里跳转到 RStata 短书平台获取附件：使用 R 语言爬取全部上市公司的年报数据
评论