TIPS program crawling

URL 만들기

basic_url + 숫자의 형태로 저장되어있음.
다만 add_url의 경우 마우스 오버 후 하나하나 넣어주는것을 권함…
DB에는 있으나 시각화 안한 경우 고려

basic_url = "http://www.jointips.or.kr/bbs/board.php?bo_table=partner&wr_id="

# url은 mannual 추가
add_urls<- c(1, 75, 3, 10, 8, 7, 12, 11, 14, 19, 20, 18, 21, 28, 
             29, 42, 37, 39, 40, 41, 36, 47, 48, 49, 50, 51, 52, 
             54, 55, 58, 59, 60, 61, 62, 63, 66, 24, 22, 57, 65, 
             64, 67, 68, 69, 70, 45, 71, 15, 2, 4, 72, 16) %>% sort()

# vector 초기화
urls <- NULL
for(i in 1:length(add_urls)) {
        urls[i] <- paste(basic_url, add_urls[i], sep = "")
}
urls <- urls %>% na.omit()
urls %>% head()

## [1] "http://www.jointips.or.kr/bbs/board.php?bo_table=partner&wr_id=1"
## [2] "http://www.jointips.or.kr/bbs/board.php?bo_table=partner&wr_id=2"
## [3] "http://www.jointips.or.kr/bbs/board.php?bo_table=partner&wr_id=3"
## [4] "http://www.jointips.or.kr/bbs/board.php?bo_table=partner&wr_id=4"
## [5] "http://www.jointips.or.kr/bbs/board.php?bo_table=partner&wr_id=7"
## [6] "http://www.jointips.or.kr/bbs/board.php?bo_table=partner&wr_id=8"

Rvest

크롤링 시작

# DF init
result1 <- data.frame()
temp_df <- data.frame()

# 갯수만큼 주관, 소개, 표 불러오기
for(i in 1:52) {
        jugwan <- urls[i] %>%
                read_html(encoding='UTF-8') %>%
                html_nodes(xpath = '//*[@id="partners"]/section/div/div/div[2]/section[1]/header/h2') %>%
                html_text()
        breif_intro <- urls[i] %>%
                read_html(encoding='UTF-8') %>%
                html_nodes(xpath = '//*[@id="partners"]/section/div/div/div[2]/section[1]/div') %>%
                html_text()
        Sys.setlocale("LC_ALL", "English") #한글 깨짐 오류잡기위한 코드
        breif_table_list <- urls[i] %>%
                read_html() %>%
                html_nodes(xpath = '//*[@id="partners"]/section/div/div/div[2]/section[2]/div/table') %>%
                html_table()
        Sys.setlocale("LC_ALL", "Korean") #한글 깨짐 오류잡기위한 코드
        breif_table <- breif_table_list[[1]]

        temp_df <- data.frame(X1 = "주관사", X2 = jugwan) %>%
                bind_rows(data.frame(X1 = "소개", X2 = breif_intro)) %>%
                bind_rows(breif_table) %>%
                tidyr::spread(X1, X2)
        result1 <- result1 %>%
                bind_rows(temp_df)
}

최종 결과