library(magrittr)
library(jsonlite)
library(stringi)
library(futile.logger)
library(utils)
library(purrr)
## 重定义 retry 函数
retry <-
  function(expr,
           isError = function(x)
             "try-error" %in% class(x),
           maxErrors = 5,
           sleep = rnorm(1, 8, 2)) {
    attempts = 0
    retval = try(eval(expr))
    while (isError(retval)) {
      attempts = attempts + 1
      if (attempts >= maxErrors) {
        msg = sprintf("retry: too many retries [[%s]]", capture.output(str(retval)))
        flog.fatal(msg)
        stop(msg)
      } else {
        msg = sprintf(
          "retry: error in attempt %i/%i [[%s]]",
          attempts,
          maxErrors,
          capture.output(str(retval))
        )
        flog.error(msg)
        warning(msg)
      }
      if (sleep > 0)
        Sys.sleep(sleep)
      retval = try(eval(expr))
    }
    return(retval)
  }


## 构造从内容到书名的函数
## 简单的想法是获取书名号中的内容
## 如果不在书名号中，则会导致失效
## 重构 read_html 和 html_text 函数，增加稳定性
read_html_beta <- function(x){
  x <- try(read_html(x))
}
html_text_beta <- function(x){
  x <- try(html_text(x))
}

Content2tittle <- function(content) {
  title <- content %>%
    purrr::map_chr('content') %>%
    map(read_html_beta) %>%
    map(html_text_beta) %>%
    stri_extract_all_regex('《.*?》') %>%
    map(unique)
  return(title)
}

raw_data <- list()
## 获取知乎的 API 接口数据
url <- 'https://www.zhihu.com/api/v4/questions/306249128/answers?data[*].author.follower_count,badge[*].topics=&data[*].mark_infos[*].url=&include=data[*].is_normal,admin_closed_comment,reward_info,is_collapsed,annotation_action,annotation_detail,collapse_reason,is_sticky,collapsed_by,suggest_edit,comment_count,can_comment,content,editable_content,voteup_count,reshipment_settings,comment_permission,created_time,updated_time,review_info,relevant_info,question,excerpt,relationship.is_authorized,is_author,voting,is_thanked,is_nothelp,is_labeled,is_recognized,paid_info,paid_info_content&limit=10&offset=0&platform=desktop&sort_by=default'

## 循环获取所有数据
i <- 1
is_end <- FALSE
while (!is_end) {
  m <- retry(readLines(url)) %>%
    fromJSON(simplifyVector = FALSE)
  content <- m$data
  next_url <- m$paging$'next' # 接下来需要解析的 url
  url <- next_url
  raw_data[[i]] <- Content2tittle(content)
  is_end <- m$paging$is_end # 利用 is_end 来判断是否结束
  cat('This is ', i, 'round', '\n')
  i <- i + 1
  Sys.sleep(runif(1, 4, 10))
}

## 踢掉一些没有信息的 answer
removed_data <- raw_data[raw_data %>% map(length) != 0]
nid <- raw_data %>% map(function(x)map(x, length)) %>% unlist()
fnid <- rep(1:length(nid), times = nid)

## 返回所有的书目
data <- data.frame( title = unlist(removed_data), id = fnid) %>%
  transform(title = as.character(title)) %>%
  na.omit() %>%
  subset(nchar(title) <= 20)

library(data.table)
data <- data.table(data)
data[, .N, title][order(-N)][N >= 10]