PythonSample/readhub.py at master · liuhao2018/PythonSample · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
# coding:utf-8
import urllib2
import urllib
import zlib
import simplejson
import time
base_url = 'https://api.readhub.me/topic'
custom_header = {
   'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64)  AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36',
   'Accept-Encoding':'gzip, deflate, sdch, br',
   'Accept-Language':'zh-CN,zh;q=0.8'
}
default_page_size = 10
last_cursor = 0
isFirstRequest = True
total_count = 0;
current_count = 0
def getData():
    start_time = time.time()
    global isFirstRequest
    global last_cursor
    global total_count
    global current_count

    if isFirstRequest:
        request = urllib2.Request(base_url+'?pageSize=1',headers=custom_header)
        isFirstRequest = False
    else:
        if last_cursor != 0:
            param = {
                'lastCursor':last_cursor,
                'pageSize':default_page_size
            }
            str_param = urllib.urlencode(param)
            target_url = base_url+'?'+str_param
            request = urllib2.Request(target_url,headers=custom_header)

    response =urllib2.urlopen(request)
    if response == None:
        print '程序出现异常 error_code == -2'
        return

    if response.info().get('Content-Encoding') == 'gzip':
        real_json = zlib.decompress(response.read(),16+zlib.MAX_WBITS);
    else :
        real_json = response.read()

    obj_data = simplejson.loads(real_json)
    if last_cursor == 0 :
        last_cursor = obj_data.get('data')[0].get('order')
        total_count = obj_data.get('totalItems')
        current_count += 1
        print str(current_count) + '  ' + obj_data.get('data')[0].get('title')

    else :
         if  len (obj_data.get('data')) > 0 :
                for news in obj_data.get('data'):
                    current_count += 1
                    print str(current_count) + '  ' + news.get('title')
                    last_cursor = obj_data.get('data')[len (obj_data.get('data')) - 1].get('order')

    if total_count >= current_count:
        getData()
    else:
        end_time = time.time()
        print '下载用时：' + str(end_time - start_time)
        print '============下载完成  人生苦短 我用Python============='


if __name__ == '__main__':
    print '正在玩命加载中...'
    getData()