使用 R 语言爬取处理 WIPO 绿色专利分类数据

绿色专利的筛选通常使用 WIPO 提供的绿色专利分类范围清单：https://www.wipo.int/classifications/ipc/green-inventory/home ，今天就让我们一起学习下如何使用 R 语言爬取这个网页上的数据，然后整理成我们需要的形式。后续课程我还会讲解如何根据这个爬取结果来筛选绿色专利。

通过网页分析，我们可以发现这个列表的数据就在这里：https://www.wipo.int/classifications/ipc/green-inventory/main.ea782575202e01049dd6.js 通过简单的手动处理，我们就可以把里面的 json 数据提取处理了：

library(jsonlite)
library(tidyverse)
fromJSON("temp.js") %>%
  as_tibble()

#> # A tibble: 7 × 6
#>   expanded folder key   title                                data$trans children
#>   <lgl>    <lgl>  <chr> <chr>                                <chr>      <list>
#> 1 TRUE     TRUE   _1    ALTERNATIVE ENERGY PRODUCTION        PRODUCTIO… <df>
#> 2 TRUE     TRUE   _98   TRANSPORTATION                       TRANSPORTS <df>
#> 3 TRUE     TRUE   _124  ENERGY CONSERVATION                  CONSERVAT… <df>
#> 4 TRUE     TRUE   _141  WASTE MANAGEMENT                     GESTION D… <df>
#> 5 TRUE     TRUE   _192  AGRICULTURE / FORESTRY               AGRICULTU… <df>
#> 6 TRUE     TRUE   _198  ADMINISTRATIVE, REGULATORY OR DESIG… ASPECTS A… <df>
#> 7 TRUE     TRUE   _202  NUCLEAR POWER GENERATION             PRODUCTIO… <df>
#> # … with 1 more variable: data$IPC <chr>

实际上从另外一个链接也可以下载到这个 json 数据：https://www.wipo.int/export/sites/www/classifications/ipc/en/green_inventory/assets/green_inventory.js 所以我们还是直接处理这个：

# https://www.wipo.int/classifications/ipc/en/green_inventory/index.html
library(tidyverse)
download.file('https://www.wipo.int/export/sites/www/classifications/ipc/en/green_inventory/assets/green_inventory.js', 'green_inventory.js')

# 处理这个数据
library(jsonlite)
read_lines('green_inventory.js') %>%
  paste0(collapse = "") %>%
  str_remove('// Created on 20190130_112331gi=') %>%
  fromJSON() %>%
  as_tibble() -> ipc

ipc

#> # A tibble: 7 × 6
#>   expanded folder key   title                                data$trans children
#>   <lgl>    <lgl>  <chr> <chr>                                <chr>      <list>
#> 1 TRUE     TRUE   _1    ALTERNATIVE ENERGY PRODUCTION        PRODUCTIO… <df>
#> 2 TRUE     TRUE   _98   TRANSPORTATION                       TRANSPORTS <df>
#> 3 TRUE     TRUE   _124  ENERGY CONSERVATION                  CONSERVAT… <df>
#> 4 TRUE     TRUE   _141  WASTE MANAGEMENT                     GESTION D… <df>
#> 5 TRUE     TRUE   _192  AGRICULTURE / FORESTRY               AGRICULTU… <df>
#> 6 TRUE     TRUE   _198  ADMINISTRATIVE, REGULATORY OR DESIG… ASPECTS A… <df>
#> 7 TRUE     TRUE   _202  NUCLEAR POWER GENERATION             PRODUCTIO… <df>
#> # … with 1 more variable: data$IPC <chr>

这样得到的 7 个 title 就是上面网页截图上的 7 个类别，我们简单处理下：

ipc %>%
  unnest(data) %>%
  select(l1key = key, l1title = title, IPC = IPC) -> l1df
l1df
#> # A tibble: 7 × 3
#>   l1key l1title                                      IPC
#>   <chr> <chr>                                        <chr>
#> 1 _1    ALTERNATIVE ENERGY PRODUCTION                ""
#> 2 _98   TRANSPORTATION                               ""
#> 3 _124  ENERGY CONSERVATION                          ""
#> 4 _141  WASTE MANAGEMENT                             ""
#> 5 _192  AGRICULTURE / FORESTRY                       ""
#> 6 _198  ADMINISTRATIVE, REGULATORY OR DESIGN ASPECTS ""
#> 7 _202  NUCLEAR POWER GENERATION                     ""

然后我们再看第二级：

ipc %>%
  unnest(data) %>%
  select(l1key = key, l1title = title, children) %>%
  unnest(children) %>%
  unnest(data) %>%
  select(l1key = l1key, l1title = l1title,
         l2key = key, l2title = title, IPC) -> l2df

l2df

#> # A tibble: 39 × 5
#>    l1key l1title                       l2key l2title                       IPC
#>    <chr> <chr>                         <chr> <chr>                         <chr>
#>  1 _1    ALTERNATIVE ENERGY PRODUCTION _2    Bio-fuels                     ""
#>  2 _1    ALTERNATIVE ENERGY PRODUCTION _11   Integrated gasification comb… "C10…
#>  3 _1    ALTERNATIVE ENERGY PRODUCTION _12   Fuel cells                    "H01…
#>  4 _1    ALTERNATIVE ENERGY PRODUCTION _17   Pyrolysis or gasification of… "C10…
#>  5 _1    ALTERNATIVE ENERGY PRODUCTION _18   Harnessing energy from manma… ""
#>  6 _1    ALTERNATIVE ENERGY PRODUCTION _33   Hydro energy                  ""
#>  7 _1    ALTERNATIVE ENERGY PRODUCTION _40   Ocean thermal energy convers… "F03…
#>  8 _1    ALTERNATIVE ENERGY PRODUCTION _41   Wind energy                   "F03…
#>  9 _1    ALTERNATIVE ENERGY PRODUCTION _47   Solar energy                  "F24…
#> 10 _1    ALTERNATIVE ENERGY PRODUCTION _74   Geothermal energy             "F24…
#> # … with 29 more rows

第三级：

ipc %>%
  unnest(data) %>%
  select(l1key = key, l1title = title, children) %>%
  unnest(children) %>%
  unnest(data) %>%
  select(l1key = l1key, l1title = l1title,
         l2key = key, l2title = title, children) %>%
  unnest(children) %>%
  unnest(data) %>%
  select(l1key = l1key, l1title = l1title,
         l2key = l2key, l2title = l2title,
         l3key = key, l3title = title, IPC) -> l3df

第四级：

ipc %>%
  unnest(data) %>%
  select(l1key = key, l1title = title, children) %>%
  unnest(children) %>%
  unnest(data) %>%
  select(l1key = l1key, l1title = l1title,
         l2key = key, l2title = title, children) %>%
  unnest(children) %>%
  unnest(data) %>%
  select(l1key = l1key, l1title = l1title,
         l2key = l2key, l2title = l2title,
         l3key = key, l3title = title, children) %>%
  unnest(children) %>%
  unnest(data) %>%
  select(l1key = l1key, l1title = l1title,
         l2key = l2key, l2title = l2title,
         l3key = l3key, l3title = l3title,
         l4key = key, l4title = title, IPC) -> l4df

第五级：

ipc %>%
  unnest(data) %>%
  select(l1key = key, l1title = title, children) %>%
  unnest(children) %>%
  unnest(data) %>%
  select(l1key = l1key, l1title = l1title,
         l2key = key, l2title = title, children) %>%
  unnest(children) %>%
  unnest(data) %>%
  select(l1key = l1key, l1title = l1title,
         l2key = l2key, l2title = l2title,
         l3key = key, l3title = title, children) %>%
  unnest(children) %>%
  unnest(data) %>%
  select(l1key = l1key, l1title = l1title,
         l2key = l2key, l2title = l2title,
         l3key = l3key, l3title = l3title,
         l4key = key, l4title = title, children) %>%
  unnest(children) %>%
  unnest(data) %>%
  select(l1key = l1key, l1title = l1title,
         l2key = l2key, l2title = l2title,
         l3key = l3key, l3title = l3title,
         l4key = l4key, l4title = l4title,
         l5key = key, l5title = title, IPC) -> l5df

合并所有的结果：

bind_rows(
  l1df %>% mutate(key = l1key),
  l2df %>% mutate(key = l2key),
  l3df %>% mutate(key = l3key),
  l4df %>% mutate(key = l4key),
  l5df %>% mutate(key = l5key)
) %>%
  select(key, IPC, everything()) %>%
  mutate(key = str_remove_all(key, "_")) %>%
  type_convert() %>%
  arrange(key) %>%
  select(-l1key, -l2key, -l3key, -l4key, -l5key) %>%
  dplyr::filter(!is.na(IPC)) %>%
  mutate(l1title = str_to_sentence(l1title)) -> ipcdf
ipcdf

#> # A tibble: 186 × 7
#>      key IPC                             l1title l2title l3title l4title l5title
#>    <dbl> <chr>                           <chr>   <chr>   <chr>   <chr>   <chr>
#>  1     3 C10L 5/00, 5/40-5/48            Altern… Bio-fu… Solid … <NA>    <NA>
#>  2     4 C10B 53/02. C10L 5/40, 9/00     Altern… Bio-fu… Solid … Torref… <NA>
#>  3     5 C10L 1/00, 1/02, 1/14           Altern… Bio-fu… Liquid… <NA>    <NA>
#>  4     6 C10L 1/02, 1/19                 Altern… Bio-fu… Liquid… Vegeta… <NA>
#>  5     7 C07C 67/00, 69/00. C10G. C10L … Altern… Bio-fu… Liquid… Biodie… <NA>
#>  6     8 C10L 1/02, 1/182. C12N 9/24. C… Altern… Bio-fu… Liquid… Bioeth… <NA>
#>  7     9 C02F 3/28, 11/04. C10L 3/00. C… Altern… Bio-fu… Biogas  <NA>    <NA>
#>  8    10 C12N 1/13, 1/15, 1/21, 5/10, 1… Altern… Bio-fu… From g… <NA>    <NA>
#>  9    11 C10L 3/00. F02C 3/28            Altern… Integr… <NA>    <NA>    <NA>
#> 10    12 H01M 4/86-4/98, 8/00-8/24, 12/… Altern… Fuel c… <NA>    <NA>    <NA>
#> # … with 176 more rows

里面的不少 IPC 使用 . 分隔范围，可以使用 tidytext 包的 unnest_tokens() 函数展开这些：

library(tidytext)
ipcdf %>%
  unnest_tokens(IPC, IPC, token = stringr::str_split,
                pattern = "\\. ", to_lower = FALSE) %>%
  mutate(class = str_sub(IPC, 1, 4),
         IPC = str_remove_all(IPC, class),
         IPC = str_trim(IPC)) %>%
  select(key, IPC, class, everything()) %>%
  unnest_tokens(IPC, IPC, token = stringr::str_split,
                pattern = ", ", to_lower = FALSE) -> ipcdf
ipcdf

#> # A tibble: 350 × 8
#>      key IPC       class l1title                 l2title l3title l4title l5title
#>    <dbl> <chr>     <chr> <chr>                   <chr>   <chr>   <chr>   <chr>
#>  1     3 5/00      C10L  Alternative energy pro… Bio-fu… Solid … <NA>    <NA>
#>  2     3 5/40-5/48 C10L  Alternative energy pro… Bio-fu… Solid … <NA>    <NA>
#>  3     4 53/02     C10B  Alternative energy pro… Bio-fu… Solid … Torref… <NA>
#>  4     4 5/40      C10L  Alternative energy pro… Bio-fu… Solid … Torref… <NA>
#>  5     4 9/00      C10L  Alternative energy pro… Bio-fu… Solid … Torref… <NA>
#>  6     5 1/00      C10L  Alternative energy pro… Bio-fu… Liquid… <NA>    <NA>
#>  7     5 1/02      C10L  Alternative energy pro… Bio-fu… Liquid… <NA>    <NA>
#>  8     5 1/14      C10L  Alternative energy pro… Bio-fu… Liquid… <NA>    <NA>
#>  9     6 1/02      C10L  Alternative energy pro… Bio-fu… Liquid… Vegeta… <NA>
#> 10     6 1/19      C10L  Alternative energy pro… Bio-fu… Liquid… Vegeta… <NA>
#> # … with 340 more rows

根据 IPC 变量，我们把数据分为下面几类：

第一类：只有大类的

ipcdf %>%
  dplyr::filter(str_length(class) == 3) -> ipcdf1
ipcdf1
#> # A tibble: 2 × 8
#>     key IPC   class l1title                  l2title     l3title l4title l5title
#>   <dbl> <chr> <chr> <chr>                    <chr>       <chr>   <chr>   <chr>
#> 1   115 ""    B61   Transportation           Rail vehic… <NA>    <NA>    <NA>
#> 2   203 ""    G21   Nuclear power generation Nuclear en… <NA>    <NA>    <NA>

第二类：只有大类小类的

ipcdf %>%
  anti_join(ipcdf1) %>%
  dplyr::filter(str_length(class) == 4 & IPC == "") -> ipcdf2
ipcdf2
#> # A tibble: 29 × 8
#>      key IPC   class l1title                     l2title l3title l4title l5title
#>    <dbl> <chr> <chr> <chr>                       <chr>   <chr>   <chr>   <chr>
#>  1     7 ""    C10G  Alternative energy product… Bio-fu… Liquid… Biodie… <NA>
#>  2    10 ""    A01H  Alternative energy product… Bio-fu… From g… <NA>    <NA>
#>  3    17 ""    C10J  Alternative energy product… Pyroly… <NA>    <NA>    <NA>
#>  4    30 ""    B09B  Alternative energy product… Harnes… Landfi… <NA>    <NA>
#>  5    36 ""    F03B  Alternative energy product… Hydro … Machin… <NA>    <NA>
#>  6    36 ""    F03C  Alternative energy product… Hydro … Machin… <NA>    <NA>
#>  7    41 ""    F03D  Alternative energy product… Wind e… <NA>    <NA>    <NA>
#>  8    47 ""    F24S  Alternative energy product… Solar … <NA>    <NA>    <NA>
#>  9    47 ""    H02S  Alternative energy product… Solar … <NA>    <NA>    <NA>
#> 10    57 ""    F24S  Alternative energy product… Solar … Use of… <NA>    <NA>
#> # … with 19 more rows

第三类：没有范围的

ipcdf %>%
  anti_join(ipcdf1) %>%
  anti_join(ipcdf2) %>%
  dplyr::filter(!str_detect(IPC, "-")) -> ipcdf3

第四类：有范围的

ipcdf %>%
  anti_join(ipcdf1) %>%
  anti_join(ipcdf2) %>%
  anti_join(ipcdf3) -> ipcdf4

对于这里用 - 连接的范围，我们需要手动找到这些范围包含的所有分类号补齐，补齐之后得到的数据是这样的：

readxl::read_xlsx("ipcdf4.xlsx") %>%
  unnest_tokens(IPC, IPC, token = stringr::str_split,
                pattern = ",", to_lower = FALSE) -> ipcdf4

最后我们再把这些 ipcdf* 保存下来：

ipcdf1 %>%
  haven::write_dta("ipcdf1.dta")
ipcdf2 %>%
  haven::write_dta("ipcdf2.dta")
ipcdf3 %>%
  haven::write_dta("ipcdf3.dta")
ipcdf4 %>%
  haven::write_dta("ipcdf4.dta")

下次课我们再讲解如何使用这些数据从专利数据中筛选绿色专利数据。

点击这里跳转到 RStata 短书平台获取附件：使用 R 语言爬取处理 WIPO 绿色专利分类数据