library(XML)

# 查看网站上的总页数，2016-11-19 日时有 1606 页
Page <- 1606
URL <- as.vector(paste0("http://www.nsfcms.org/index.php?r=search/index&Projects_page=",
                        1:Page))

Natural <- data.frame()
i <- 1
for (i in 1:Page){
  D1 <- as.data.frame(readHTMLTable(URL[i]))
  Natural <- rbind(Natural, D1)
  print(i)
  i <- i + 1
}

names(Natural) <- c("项目批准号", "项目类别", "项目名称", "依托单位",
                    "负责人", "金额", "起止年月","总评")

# 保存时最好以抓取日期命名
write.csv(Natural, "natural-2016-11-19.csv", row.names = F)

library(readr)
library(stringr)
library(lubridate)
library(ggplot2)
library(ggthemes)
library(knitr)
library(dplyr)

Natural <- read_csv("natural-2016-11-19.csv")

# 对字符串进行拆分
Period <- str_split(Natural$起止年月,"-", simplify = T)

# R 无法处理没有日期的数据，因此加上日期
Begin <- paste0(Period[,1], ".01")
Begin <- as.Date.character(Begin,"%Y.%m.%d")
End <- paste0(Period[,2], ".01")
End <- as.Date(End, "%Y.%m.%d")
Interval <- round((End - Begin) / (365/12)) + 1
Natural <- cbind(Natural, Begin, End, Interval)

ggplot(Natural, aes(year(Begin))) +
  geom_bar(aes(fill = 项目类别)) +
  labs(
    title = "国家自然科学基金年度项目数量（按起始年份）",
    subtitle= "按照项目类别标示",
    x = "年份",
    y = "项目数量"
  ) +
  scale_x_continuous(breaks = seq(1992, 2017, by = 1)) +
  theme_pander() +
  scale_fill_pander() +
  theme_grey(base_family = "STKaiti") +
  theme(axis.text.x  = element_text(angle = 60, vjust = 0.5))

ggplot(Natural, aes(year(Begin))) +
  geom_bar(aes(weight = 金额, fill = 项目类别)) +
  labs(
    title = "国家自然科学基金年度项目总额（按起始年份）",
    subtitle= "按照项目类别标示",
    x = "年份",
    y = "项目总额"
  ) +
  scale_x_continuous(breaks = seq(1992, 2017, by = 1)) +
  theme_pander() +
  scale_fill_pander() +
  theme_grey(base_family = "STKaiti") +
  theme(axis.text.x  = element_text(angle = 60, vjust = 0.5)) 

ggplot(Natural, aes(year(Begin))) +
  geom_bar(aes(fill = 项目类别), position = "fill") +
  labs(
    title = "国家自然科学基金年度项目类型数量占比（按起始年份）",
    x = "年份",
    y = "比例"
  ) +
  scale_x_continuous(breaks = seq(1992, 2017, by = 1)) +
  theme_pander() +
  scale_fill_pander() +
  theme_grey(base_family = "STKaiti") +
  theme(axis.text.x  = element_text(angle = 60, vjust = 0.5))

Type <- summarise(group_by(Natural, 项目类别),
                  项目总金额 = sum(金额),
                  项目金额占比 = sum(金额) / sum(Natural$金额) * 100,
                  项目数量 = length(金额),
                  项目数量占比 = length(金额) / length(Natural$项目名称) * 100,
                  项目平均金额 = mean(金额))
Type <- Type[with(Type, order(-项目总金额)), ]
kable(Type, align = "c", digits = 1)

Type_rate <- as.array(table(Natural$项目类别, Natural$总评))
Type_rate <- round(prop.table(Type_rate, margin = 1) * 100, digits = 1)
kable(Type_rate)

ggplot(Natural, aes(year(Begin))) +
  geom_bar(aes(fill = 总评), position = "fill") +
  labs(
    title = "国家自然科学基金年度项目评估情况（按起始年份）",
    subtitle= "按项目数量计算",
    x = "年份",
    y = "项目数量"
  ) +
  scale_x_continuous(breaks = seq(1992, 2017, by = 1)) +
  theme_pander() +
  scale_fill_pander() +
  theme_grey(base_family = "STKaiti") +
  theme(axis.text.x  = element_text(angle = 60, vjust = 0.5))

Bad <- as.data.frame(table(Natural[Natural$总评 == "差", 4]))
kable(Bad[Bad$Freq > 1, ], align = "c")

Rank50 <- summarise(group_by(Natural, 依托单位),
                    累计金额 = sum(金额),
                    项目数量 = length(金额),
                    项目平均金额 = mean(金额))
Rank50 <- Rank50[with(Rank50, order(-累计金额)), ]
kable(head(Rank50,50), align = "c", row.names = T)

Rank <- list()
j = 1
for(i in 1992:2017){
  D <- Natural[year(Natural$Begin) == i,]
  Rank[[j]] <- names(head(sort(tapply(D$金额, D$依托单位, sum), decreasing = T),10))
  i <- i + 1
  j <- j + 1
}
Rank <- as.data.frame(Rank)
names(Rank) <- 1992:2017
Rank_count <- table(unlist(Rank))
Rank_count_t <- as.data.frame(sort(Rank_count[Rank_count >1], decreasing = T))
names(Rank_count_t) <- c("依托单位", "进入前十次数")
kable(Rank_count_t, align = "c", row.names = T)

library(jiebaR)
W <- worker()
Title <- W[Natural$项目名称]
head(sort(table(unlist(Title)), decreasing = T), 100)