FetchFigureOneData/queryCore.py at master · chenxiing/FetchFigureOneData · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
# -*- coding: utf-8 -*-

import cookielib
import urllib2
import urllib
import httplib
import urlparse


'''网络数据查询实现类，负责网络数据交互，包括登陆和JSON数据获取'''
class QueryCore():
    def __init__(self, ctr):
        self.ctr = ctr
        self.setCookies()

    """ 设置cookie"""
    def setCookies(self):
        self.cookieFile = "cookies.txt"
        self.cookies = cookielib.MozillaCookieJar(self.cookieFile)

        try:
            self.cookies.load(ignore_discard = True, ignore_expires = True)
        except Exception:
            self.saveCookie()

        self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cookies))
        urllib2.install_opener(self.opener)

        try:
            response = self.opener.open(urllib2.Request('https://app.figure1.com'))
            response.read()
        except:
            pass
        self.saveCookie()

    """保存cookie"""
    def saveCookie(self):
        self.cookies.save(self.cookieFile, ignore_discard = True, ignore_expires = True)

    """获得全部cookie数据"""
    def readAllCookie(self):
        cookieMap = {}
        if self.cookies is not None:
            for cookie in self.cookies:
                one = cookie.name + '=' + str(cookie.value) + ('; expires=' + str(cookie.expires) if cookie.expires is not None else '' ) + '; domain=' + str(cookie.domain) + '; path=' + str(cookie.path)
                cookieMap.update({cookie.name: one})
        return cookieMap

    """更新cookie"""
    def updateCookie(self):
        self.saveCookie()
        self.cookies.load(ignore_discard = True, ignore_expires = True)
        self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cookies))
        urllib2.install_opener(self.opener)

    '''异步登陆Figure1，实际上直接用login就行了'''
    def loginAync(self, postData):
        def run(self, postData):
            try:
                url = 'https://app.figure1.com/login'
                req = urllib2.Request(url, urllib.urlencode(postData))

                req.add_header('Host', 'app.figure1.com')
                req.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36')
                req.add_header('Referer', 'https://app.figure1.com/login')
                req.add_header('Connection', 'keep-alive')
                req.add_header('X-Requested-With', 'XMLHttpRequest')
                req.add_header('Content-Type', 'application/x-www-form-urlencoded')

                response = urllib2.urlopen(req)
                #print response.read()

                return {'s': True, 'r': response.read()}
            except:
                return {'s': False, 'r': ''}

        re = run(self, postData)

        while not re['s']:
            re = run(self, postData)

        ''' fix-me! 这里还要对返回的re['r']做处理，判断是否登陆成功，由于本程序有特定用途，这里就不做太多判断了。'''
        return True

    """用户登陆，返回登陆状态码"""
    def login(self, loginData):
        if self.loginAync(loginData):
            try:
                '''
                url = "https://app.figure1.com/login"
                req = urllib2.Request(url, urllib.urlencode(loginData))

                req.add_header('Host', 'app.figure1.com')
                req.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36')
                req.add_header('Referer', 'https://app.figure1.com/login')
                req.add_header('Connection', 'keep-alive')
                req.add_header('X-Requested-With', 'XMLHttpRequest')
                req.add_header('Content-Type', 'application/x-www-form-urlencoded')

                response = urllib2.urlopen(req)
                '''
            except:
                return False

            return True
        else:
            return False


    '''获取Figure1首页JSON。pageSize为每页面显示数据条数，skip为页码。目前数据量为16500左右。'''
    def getHomeData(self, pageSize, skip):
        def run(self):
            try:
                url = 'https://app.figure1.com/s/feeds/home?pageSize=' + str(pageSize) + '&skip=' + str(skip)
                req = urllib2.Request(url)
                req.add_header('Host', 'app.figure1.com')
                req.add_header('User-Agent', 'Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.120 Safari/537.36')
                req.add_header('Referer', 'https://app.figure1.com/login')
                req.add_header('Connection', 'keep-alive')
                req.add_header('X-Requested-With', 'XMLHttpRequest')
                req.add_header('Content-Type', 'application/x-www-form-urlencoded')

                response = urllib2.urlopen(req)
                #self.updateCookie()
                print response.read()

                return {'s': True, 'r': response.read().decode()}
            except:
                return {'s': False, 'r': ''}

        re = run(self)

        while not re['s']:
            re = run(self)

        return True