python爬取微信公众号文章的方法

TIMEOUT = 20from requests import Request, Session, PreparedRequestimport requestsfrom selenium import webdriverfrom selenium.common.exceptions import NoSuchElementExceptionfrom bs4 import BeautifulSoup as bsimport pymysql # 要爬取的内容keyword = '美女图片'options = webdriver.ChromeOptions()# 设置中文options.add_argument('lang=zh_CN.UTF-8')# 更换头部options.add_argument( 'user-agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'')browser = webdriver.Chrome(chrome_options=options)REDIS_HOST = '192.168.1.248'REDIS_PORT = 6379REDIS_PASSWORD = '*****'REDIS_KEY = 'requests'PROXY_POOL_URL = 'http://127.0.0.1:8080/random'MAX_FAILED_TIME = 5 MYSQL_HOST = 'localhost'MYSQL_PORT = 3306MYSQL_USER = 'moxiao'MYSQL_PASSWORD = '******' class mysqlConn(): def __init__(self, host=MYSQL_HOST, username=MYSQL_USER, password=MYSQL_PASSWORD, port=MYSQL_PORT): ''' mysql 初始化 :param host: :param username: :param password: :param port: ''' try: self.db = pymysql.Connection(host=host, user=username, password=password, database='weixin_data', port=port) self.cursor = self.db.cursor() except pymysql.MySQLError as e: print(e.args) def insert(self, table, data): keys = ', '.join(data.keys()) values = ', '.join(['%s'] * len(data)) sql = 'insert into %s (%s) values (%s)' % (table, keys, values) try: self.cursor.execute(sql, tuple(data.values())) self.db.commit() except pymysql.MySQLError as e: print(e.args) self.db.rollback() class WeixinRequest(Request): def __init__(self, url, callback, method='GET', headers=None, need_proxy=False, fail_time=0, timeout=TIMEOUT): super(WeixinRequest, self).__init__(url=url, method=method, headers=headers) self.callback = callback self.need_proxy = need_proxy self.fail_time = fail_time self.timeout = timeout def prepare(self): p = PreparedRequest() p.prepare( method=self.method, url=self.url, headers=self.headers, ) return p class WeixinResponse(): def __init__(self, text): self.text = text def set_status_code(self, status_code): self.status_code = status_code import picklefrom redis import StrictRedis class RedisQueue(): def __init__(self): ''' 初始化redis ''' self.db = StrictRedis(host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db=3) def add(self, request): ''' 向队列添加序列化后的Request :param request:请求对象 :return:添加结果 ''' if isinstance(request, WeixinRequest): return self.db.rpush(REDIS_KEY, pickle.dumps(request)) return False def pop(self): ''' 取出下一个request并反序列化 :return: Request 或者 None ''' if self.db.llen(REDIS_KEY): return pickle.loads(self.db.lpop(REDIS_KEY)) return False def empty(self): return self.db.llen(REDIS_KEY) == 0 def del_all(self): return self.db.delete(REDIS_KEY) def get_proxy(self): ''' 从代理池获取代理IP :return: ''' try: response = requests.get(PROXY_POOL_URL) if response.status_code == 200: print('get Proxy', response.text) return response.text except requests.ConnectionError: return None from urllib.parse import urlencodefrom requests import ReadTimeout, ConnectionErrorfrom pyquery import PyQuery as pq VALD_STATUES = [200] class Spider(): base_url = 'http://weixin.sogou.com/weixin?' # 这里的page可以修改，即第几页，我本来想获取所有的个数再除以10 这样就能爬完了，但是我只是测试所以这里并没有做 # 但如果需要做可以加到schedule方法的while循环内的最下面 即self.params['page']+=1 params = {'type': 2, 's_from': 'input', 'query': keyword, 'page': 1, 'ie': 'utf8', '_sug_': 'n', '_sug_type_': ''} headers = {'Host': 'weixin.sogou.com', 'Connection': 'keep-alive', 'Cache-Control': 'max-age=0', 'Upgrade-Insecure-Requests': '1', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8', 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'zh-CN,zh;q=0.9', 'Referer': 'http: // weixin.sogou.com /', 'Cookie': '你的cookie'} # TODO 不可能把我的给你撒 session = Session() queue = RedisQueue() queue.del_all() mysql = mysqlConn() def start(self): ''' 初始化工作 :return: ''' # 全局更新headers # 如果你试过用这个方法修改headers，那么就知道这个在这里好像没什么用，我在这里浪费了至少两个小时 self.session.headers.update(self.headers) start_url = self.base_url + urlencode(self.params) # 这里我将need_proxy=False设为了False 即并没有使用代理 ps：我也就是测试一下 # 真正修改了headers是在这里 weixin_request = WeixinRequest(url=start_url, callback=self.parse_index, headers=self.headers, need_proxy=False) # 调度第一个请求 self.queue.add(weixin_request) def schedule(self): ''' 调度请求 :return: ''' while not self.queue.empty(): weixin_request = self.queue.pop() callback = weixin_request.callback print('Schedule', weixin_request.url) response = self.request(weixin_request) if response and response.status_code in VALD_STATUES: results = list(callback(response)) if results: for result in results: print('New Result', result) if isinstance(result, WeixinRequest): # 将新的文章详情的url也加入队列 self.queue.add(result) if isinstance(result, dict): # 储存到mysql self.mysql.insert('articles', result) else: self.error(weixin_request) else: self.error(weixin_request) def request(self, weixin_request): ''' 执行请求 :param weixin_request:请求 :return: 响应 ''' if not 'http://mp.weixin.qq.com/s?src' in weixin_request.url: try: if weixin_request.need_proxy: proxy = self.queue.get_proxy() if proxy: proxies = { 'http': 'http://' + proxy, 'https': 'https://' + proxy } return self.session.send(weixin_request.prepare(), timeout=weixin_request.timeout, allow_redirects=False, proxies=proxies) return self.session.send(weixin_request.prepare(), timeout=weixin_request.timeout, allow_redirects=False) except (ConnectionError, ReadTimeout) as e: print(e.args) return False else: print('-' * 20) browser.get(weixin_request.url) try: browser.find_element_by_class_name('rich_media_area_primary_inner') wr = WeixinResponse(browser.page_source) wr.set_status_code(200) return wr except NoSuchElementException: wr = WeixinResponse('') wr.set_status_code(403) return wr def parse_index(self, response): ''' 解析索引页 :param response: 响应 :return: 新的响应 ''' doc = pq(response.text) items = doc('.news-box .news-list li .txt-box h3 a').items() for item in items: url = item.attr('href') weixin_request = WeixinRequest(url=url, callback=self.parse_detail) yield weixin_request def parse_detail(self, response): ''' 解析详情页 :param response: 响应 :return: 微信公众号文章 ''' doc = pq(response.text) profile_inner = doc('.profile_inner') data = { 'title': doc('.rich_media_title').text(), 'content': doc('.rich_media_content').text(), 'date': doc('#publish_time').text(), # 'nickname':doc('#js_profile_qrcode > div > strong').text(), 'nickname': profile_inner.find('.profile_nickname').text(), 'wechat': [ns for ns in profile_inner.find('.profile_meta').find('.profile_meta_value').items()][ 0].text() } # 储存图片 print('#' * 30) soup = bs(response.text) wn = soup.find_all('img') for img in wn: if img.has_attr('_width') and img.has_attr('data-src'): print(img.attrs['data-src']) yield data def error(self, weixin_request): ''' 错误处理 :param weixin_request:请求 :return: ''' weixin_request.fail_time = weixin_request.fail_time + 1 print('Request Failed', weixin_request.fail_time, 'Times', weixin_request.url) if weixin_request.fail_time < MAX_FAILED_TIME: self.queue.add(weixin_request) def run(self): self.start() self.schedule() if __name__ == '__main__': spider = Spider() spider.run()