DoubanBookSpider/getDoubanBooks.py at main · Jolanyu/DoubanBookSpider · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
# coding=UTF-8
from bs4 import BeautifulSoup
import re
import pandas as pd
import time
import requests
import urllib.request


def getHtml(url):
    try:
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.93 Safari/537.36 Edg/96.0.1054.53'
        }
        page = urllib.request.Request(url, headers=headers)
        page = urllib.request.urlopen(page)
        html = page.read()
    except:
        print("failed to geturl")
        return ''
    else:
        return html


def parsePage(html, str, flag=True):
    try:
        if flag:
            reStr = r'<span class="pl">'+str+r'.*?</span>(.*?)<br/>'
        else:
            reStr = r'<span class="pl">.*?'+str+r'.*?">(.*?)</a>'
        infoList = re.findall(reStr, html, re.S)
        if len(infoList) >= 1:
            return infoList[0].strip().replace("\n", "").replace(" ", "")
        else:
            return None
    except Exception as e:
        print(e)
        return None


def informationToCsv(html, bookClass):
    try:
        soup = BeautifulSoup(
            html, features="lxml")
        info = str(soup.select('#info')[0])
        pList = None
        summary = ""
        try:
            pList = soup.select('.intro')[0].find_all("p")
            for p in pList:
                summary = summary + p.string
            summary = summary.replace("\r", "")
            summary = summary.replace("(展开全部)", "")
        except Exception as e:
            print(e)
        title = soup.h1.span.string
        image = soup.select('#mainpic')[0].a['href']
        author = parsePage(info, "作者", False)
        publisher = parsePage(info, "出版社")
        translator = parsePage(info, "译者", False)
        isbn = parsePage(info, "ISBN")
        timeStr = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())

        bookDict = {}
        bookDict['book_isbn'] = isbn
        bookDict['bokk_title'] = title
        bookDict['book_image'] = image
        bookDict['book_author'] = author
        bookDict['book_translator'] = translator
        bookDict['book_category'] = bookClass
        bookDict['book_publisher'] = publisher
        bookDict['book_summary'] = summary
        bookDict['create_time'] = timeStr
        bookDict['update_time'] = timeStr
        print(bookDict)
        bookList = []
        bookList.append(bookDict)
        bookDataFrame = pd.DataFrame(bookList)
        csvName = bookClass + ".csv"
        bookDataFrame.to_csv(csvName, mode='a', header=False,
                             index=False, encoding='utf-8')
    except Exception as e:
        print(e)


def getBookUrl(html):
    soup = BeautifulSoup(
        html, features="lxml")
    aList = soup.select(".nbg")
    urlList = []
    for item in aList:
        urlList.append(item['href'])
    return urlList


# https://book.douban.com/tag/%E5%84%BF%E7%AB%A5%E6%96%87%E5%AD%A6?start=0&type=S
if __name__ == "__main__":
    startPage = 0
    endPage = 49
    bookClass = "动物"
    bookClassCode = urllib.parse.quote(bookClass)
    for i in range(startPage, endPage+1):
        url = "https://book.douban.com/tag/"+bookClassCode+"?start=" + \
            str(i*20)+"&type=S"
        print("正在爬第"+str(i+1)+"页 url:"+url)
        html = getHtml(url)
        urlList = getBookUrl(html)
        print(urlList)
        time.sleep(5)
        for url in urlList:
            time.sleep(5)
            html = getHtml(url)
            informationToCsv(html, bookClass)