library(quanteda)

Download corpus

Download corpus constructed from Report on the Work of the Government published by Premier of the State Council between 1954 and 2017. You can download the corpus using the quanteda.corpora package.

# read text files
devtools::install_github("quanteda/quanteda.corpora")

library(quanteda.corpora)
corp <- quanteda.corpora::download(url = "https://www.dropbox.com/s/37ojd5knz1qeyul/data_corpus_chinesegovreport.rds?dl=1")

Tokenization

# Chinese stopwords
ch_stop <- stopwords("zh", source = "misc")

# tokenize
ch_toks <- 
    tokens(corp, remove_punct = TRUE) %>%
    tokens_remove(ch_stop)

# construct a dfm
ch_dfm <- dfm(ch_toks)
topfeatures(ch_dfm)

## 发展 经济 社会 建设 改革 人民 主义 工作 企业 国家 
## 5627 5036 4255 4248 2931 2897 2817 2642 2627 2595

Analysis

Word cloud

# plot a word cloud
set.seed(100)

# to set the font correctly for macOS
textplot_wordcloud(ch_dfm, min_count = 500, random_order = FALSE,
                   rotation = .25, max_words = 100,
                   min_size = 0.5, max_size = 2.8,
                   font = if (Sys.info()['sysname'] == "Darwin") "SimHei" else NULL,
                   color = RColorBrewer::brewer.pal(8,"Dark2"))

Feature co-occurrence matrix

# fcm within the window size of 5
ch17_corp <- corpus_subset(corp, Year == "2017")
ch17_toks <- 
    tokens(ch17_corp, remove_punct = TRUE) %>% 
    tokens_remove(ch_stop)
ch_fcm <- fcm(ch17_toks, context = "window")  
topfeatures(ch_fcm["改革", ])

## 推进 制度   性 体制 完善   试   点 供给   侧 结构 
##   23   19   11   11   10    8    8    7    7    7

Unsupervised document scaling

wf <- textmodel_wordfish(ch_dfm)
y <- 1954:2017
y <- y[y <= 1964 | y >= 1975]
y <- y[!y %in% c(1963, 1961, 1962, 1976, 1977)]
plot(y, wf$theta, xlab = "Year", ylab = "Position")

Collocations

# bigrams cross the whole dataset
ch_col <- textstat_collocations(ch_toks, size = 2, min_count = 20)
knitr::kable(head(ch_col, 10))

collocation	count	length	lambda	z
社会主义	1787	2	5.661200	128.61741
亿元	689	2	7.445259	93.01487
现代化	632	2	6.950471	83.55186
体制改革	504	2	5.193066	77.37908
五年计划	341	2	5.359068	71.64727
各级政府	306	2	6.110596	66.63454
增长百分	300	2	5.520761	65.88051
万吨	212	2	6.589957	62.56344
国民经济	589	2	6.014672	61.80426
充分发挥	191	2	6.584129	61.24885

# bigrams in 2017 report
ch17_col <- textstat_collocations(ch17_toks, size = 2)
knitr::kable(head(ch17_col, 10))

collocation	count	length	lambda	z
人民群众	12	2	5.406485	12.89405
亿元	14	2	8.302483	12.62130
调控	11	2	7.593472	12.41243
政府工作	9	2	4.709869	11.07905
深入实施	8	2	5.018234	10.92377
党中央	7	2	5.746878	10.90837
体制改革	11	2	5.317035	10.53518
国内生产	6	2	6.166520	10.48816
现代化	8	2	5.705688	10.43435
基础设施	7	2	7.549273	10.42465

Example: Chinese text analysis

Haiyan Wang