使用代理池处理反爬抓取微信文章
搜狗(http://weixin.sogou.com/)
已经为我们做了一层微信文章的爬取,通过它我们可以获取一些微信文章的列表以及微信公众号的一些信息,
但是它有很多反爬虫的措施,可以检测到你的IP异常,然后把你封掉。
本文采用代理的方法处理反爬来抓取微信文章。
用到的
- urllib
- lxml
- pyquery
- requests
- re
- pymongo
- sys
代理池
效果图
实现
# -*- coding: utf-8 -*-
"""
@Time : 2021/1/27 12:29
@Auth : Ne-21
@File :weixin_spider.py
@IDE :PyCharm
@Motto:Another me.
"""
from urllib.parse import urlencode
from lxml.etree import XMLSyntaxError
from pyquery import PyQuery as pq
import requests
import re
import pymongo
import sys
sys.setrecursionlimit(3000) # 将默认的递归深度修改为3000
base_url = 'https://weixin.sogou.com/weixin?'
headers = {
'Referer': 'https://weixin.sogou.com/weixin?query=%E9%A3%8E%E6%99%AF&type=2&page=1&ie=utf8',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'same-origin',
'Sec-Fetch-User': '?1',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 '
'Safari/537.36',
'Host': 'weixin.sogou.com',
'Cookie': ''
}
keyword = '云计算'
# 获取、设置全局代理
proxy_pool_url = 'http://127.0.0.1:5555/random'
proxy = None
# 最大请求次数
max_count = 5
# MongoDB
MONGO_URL = 'localhost'
MONGO_DB = 'weixin'
MONGO_TABLE = 'articles'
client = pymongo.MongoClient(MONGO_URL)
db = client[MONGO_DB]
# 获取代理信息
def get_proxy():
try:
response = requests.get(proxy_pool_url)
if response.status_code == 200:
return response.text
return None
except ConnectionError:
return None
# 搜索
def get_index(keyword, page):
data = {
'query': keyword,
'page': page,
'type': '2'
}
queries = urlencode(data)
url = base_url + queries
html = get_html(url)
return html
# 获取搜索页面信息
def get_html(url, count=1):
print('请求目标网址', url)
print('当前请求次数', count)
global proxy
# 判断请求次数
if count == max_count:
print('请求次数过多')
return None
try:
# 是否有代理
if proxy:
proxies = {
'http': 'http://' + proxy
}
response = requests.get(url, headers=headers, allow_redirects=False, proxies=proxies)
else:
# allow_redirects=False 不处理自动跳转
response = requests.get(url, headers=headers, allow_redirects=False)
if response.status_code == 200:
return response.text
if response.status_code == 302:
# need proxy
print('302')
proxy = get_proxy()
if proxy:
print('Using Proxy', proxy)
# count += 1
return get_html(url)
else:
print('Get Proxy Failed')
return None
except ConnectionError as e:
print('Error', e.args)
proxy = get_proxy()
count += 1
return get_html(url, count)
# 解析搜索页面
def parse_index(html):
doc = pq(html)
items = doc('.news-box .news-list li .txt-box h3 a').items()
for item in items:
yield str('https://weixin.sogou.com') + item.attr('href')
# 获取真实的文章地址
def get_detail_true_url(url):
global proxy
try:
if proxy:
proxies = {
'http': 'http://' + str(proxy)
}
response = requests.get(url, headers=headers, allow_redirects=False, proxies=proxies)
else:
# allow_redirects=False 不处理自动跳转
response = requests.get(url, headers=headers, allow_redirects=False)
if response.status_code == 200:
response = response.text
url_param = re.compile("url \+= '(.*?)';", re.S).findall(response)
true_url = ''
for i in url_param:
true_url += i
true_url.replace('@', '')
return true_url
except:
return get_detail_true_url(url)
# 获取文章详情
def get_detail(true_url):
try:
response = requests.get(true_url)
if response.status_code == 200:
response = response.text
return response
return None
except ConnectionError:
return None
# 解析文章
def parse_detail(html):
try:
doc = pq(html)
title = doc('.rich_media_title').text()
content = doc('.rich_media_content').text()
date = doc('#publish_time').text()
nickname = doc('#js_name').text()
wechat = doc('#js_profile_qrcode > div > p:nth-child(3) > span').text()
return {
'title': title,
'content': content,
'date': date,
'nickname': nickname,
'wechat': wechat
}
except XMLSyntaxError:
return None
def save_to_mongo(data):
# 如果有一样的,更新,没有,插入
if db[MONGO_TABLE].update({'title': data['title']}, {'$set': data}, True):
print('存储到数据库成功', data['title'])
else:
print('存储失败', data['title'])
def main():
for page in range(1, 100):
html = get_index(keyword, page)
if html:
article_urls = parse_index(html)
for article_url in article_urls:
article_html = get_detail_true_url(article_url)
article_true_html = get_detail(article_html)
if article_true_html:
article_data = parse_detail(article_true_html)
print(article_data)
if article_data:
save_to_mongo(article_data)
if __name__ == '__main__':
main()
版权声明:
作者:Ne-21
链接:https://blog.gocos.cn/archives/61.html
来源:云淡风轻
文章版权归作者所有,未经允许请勿转载。
THE END
0
二维码
海报
使用代理池处理反爬抓取微信文章
搜狗(http://weixin.sogou.com/)
已经为我们做了一层微信文章的爬取,通过它我们可以获取一些微信文章的列表以及微信公众号的一些信息,
但是它有很多反爬……

共有 0 条评论