|
Python3：爬取新浪、网易、今日头条、UC四大网站新闻标题及内容
以爬取相应网站的社会新闻内容为例：
一、新浪：
新浪网的新闻比较好爬取，我是用BeautifulSoup直接解析的，它并没有使用JS异步加载，直接爬取就行了。
'''
新浪新闻：http://news.sina.com.cn/society/
Date：20180920
Author：lizm
Description：获取新浪新闻
'''
import requests
from bs4 import BeautifulSoup
from urllib import request
import sys
import re
import os
def getNews(title,url,m):
Hostreferer = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
}
req = request.Request(url)
response = request.urlopen(req)
#过滤非utf-8的网页新闻
response = response.read().decode('utf-8',"ignore")
soup = BeautifulSoup(response,'lxml')
tag = soup.find('div',class_='article')
if tag == None:
return 0
#获取文章发布时间
fb_date = soup.find('div','date-source').span.string
#获取发布网站名称
fb_www= soup.find('div','date-source').a.string
#获取文章内容
rep = re.compile("[\s+\.\!\/_,$%^*(+\"\']+|[+?、~*（）]+")
title = rep.sub('',title)
title = title.replace(':','：')
filename = sys.path[0]+"/news/"+title+".txt"
with open(filename,'w',encoding='utf8') as file_object:
file_object.write(fb_date + " " + fb_www)
file_object.write("\n")
file_object.write("网址:"+url)
file_object.write("\n")
file_object.write(title)
file_object.write(tag.get_text())
i = 0
for image in tag.find_all('div','img_wrapper'):
title_img = title +str(i)
#保存图片
#判断目录是否存在
if (os.path.exists(sys.path[0]+"/news/"+title)):
pass
else:
#不存在，则新建目录
os.mkdir(sys.path[0]+"/news/"+title)
os.chdir(sys.path[0]+"/news/"+title)
file_name = "http://news.sina.com.cn/"+image.img.get('src').replace('//','')
html = requests.get(file_name, headers=Hostreferer)
# 图片不是文本文件，以二进制格式写入，所以是html.content
title_img = title_img +".jpg"
f = open(title_img, 'wb')
f.write(html.content)
f.close()
i+=1
print('成功爬取第', m,'个新闻',title)
return 0
#获取社会新闻（最新的162条新闻）
def getTitle(url):
req = request.Request(url)
response = request.urlopen(req)
response = response.read().decode('utf8')
soup = BeautifulSoup(response,'lxml')
y = 0
for tag in soup.find('ul',class_='seo_data_list').find_all('li'):
if tag.a != None:
#if y== 27:
print(y,tag.a.string,tag.a.get('href'))
temp = tag.a.string
getNews(temp,tag.a.get('href'),y)
y += 1
if __name__ == '__main__':
url = 'http://news.sina.com.cn/society/'
getTitle(url)
二、网易：
网易新闻的标题及内容是使用js异步加载的，单纯的下载网页源代码是没有标题及内容的，我们可以在Network的js中找到我们需要的内容，这里我使用了正则表达式来获取我们需要的标题及其链接，并使用了BeautifulSoup来获取相应标题的内容。
import re
from urllib import request
from bs4 import BeautifulSoup
def download(title, url):
req = request.urlopen(url)
res = req.read()
soup = BeautifulSoup(res,'lxml')
#print(soup.prettify())
tag = soup.find('div',class_='post_text')
#print(tag.get_text())
title = title.replace(':','')
title = title.replace('"','')
title = title.replace('|','')
title = title.replace('/','')
title = title.replace('\\','')
title = title.replace('*','')
title = title.replace('','')
title = title.replace('?','')
#print(title)
file_name = r'D:\code\python\spider_news\NetEase_news\sociaty\\' +title + '.txt'
file = open(file_name,'w',encoding = 'utf-8')
file.write(tag.get_text())
if __name__ == '__main__':
urls = ['http://temp.163.com/special/00804KVA/cm_shehui.js?callback=data_callback',
'http://temp.163.com/special/00804KVA/cm_shehui_02.js?callback=data_callback',
'http://temp.163.com/special/00804KVA/cm_shehui_03.js?callback=data_callback']
for url in urls:
#url = 'http://temp.163.com/special/00804KVA/cm_shehui_02.js?callback=data_callback'
req = request.urlopen(url)
res = req.read().decode('gbk')
#print(res)
pat1 = r'"title":"(.*?)",'
pat2 = r'"tlink":"(.*?)",'
m1 = re.findall(pat1,res)
news_title = []
for i in m1:
news_title.append(i)
m2 = re.findall(pat2,res)
news_url = []
for j in m2:
news_url.append(j)
for i in range(0,len(news_url)):
#print(news_title[i],news_body[i])
download(news_title[i],news_url[i])
print('正在爬取第' + str(i) + '个新闻',news_title[i])
三、头条：
头条的新闻跟前两个也都不一样，它的标题和链接是封装到json文件中的，但是他json文件的url参数是通过一个js随机算法变化的，所以我们需要模拟json文件的参数，否则我们找不到json文件的具体url，我是通过http://www.jianshu.com/p/5a93673ce1c0这篇博客才了解到url获取方法的，而且也解决了总是下载重复新闻的问题，该网站自带反爬机制，需要添加cookie。关于新闻的内容，我用了正则表达式提取了中文。
from urllib import request
import requests
import json
import time
import math
import hashlib
import re
from bs4 import BeautifulSoup
def get_url(max_behot_time, AS, CP):
url = 'https://www.toutiao.com/api/pc/feed/?category=news_society&utm_source=toutiao&widen=1' \
'&max_behot_time={0}' \
'&max_behot_time_tmp={0}' \
'&tadrequire=true' \
'&as={1}' \
'&cp={2}'.format(max_behot_time, AS, CP)
return url
def get_ASCP():
t = int(math.floor(time.time()))
e = hex(t).upper()[2:]
m = hashlib.md5()
m.update(str(t).encode(encoding='utf-8'))
i = m.hexdigest().upper()
if len(e) != 8:
AS = '479BB4B7254C150'
CP = '7E0AC8874BB0985'
return AS,CP
n = i[0:5]
a = i[-5:]
s = ''
r = ''
for o in range(5):
s += n[o] + e[o]
r += e[o + 3] + a[o]
AS = 'AL'+ s + e[-3:]
CP = e[0:3] + r + 'E1'
# print("AS:"+ AS,"CP:" + CP)
return AS,CP
def download(title, news_url):
# print('正在爬')
req = request.urlopen(news_url)
if req.getcode() != 200:
return 0
res = req.read().decode('utf-8')
#print(res)
pat1 = r'content:(.*?),'
pat2 = re.compile('[\u4e00-\u9fa5]+')
result1 = re.findall(pat1,res)
#print(len(result1))
if len(result1) == 0:
return 0
print(result1)
result2 = re.findall(pat2,str(result1))
result3 = []
for i in result2:
if i not in result3:
result3.append(i)
#print(result2)
title = title.replace(':','')
title = title.replace('"','')
title = title.replace('|','')
title = title.replace('/','')
title = title.replace('\\','')
title = title.replace('*','')
title = title.replace('','')
title = title.replace('?','')
with open(r'D:\code\python\spider_news\Toutiao_news\society\\' + title + '.txt','w') as file_object:
file_object.write('\t\t\t\t')
file_object.write(title)
file_object.write('\n')
file_object.write('该新闻地址：')
file_object.write(news_url)
file_object.write('\n')
for i in result3:
#print(i)
file_object.write(i)
file_object.write('\n')
# file_object.write(tag.get_text())
#print('正在爬取')
def get_item(url):
#time.sleep(5)
cookies = {'tt_webid': '6478612551432734221'}
wbdata = requests.get(url,cookies = cookies)
wbdata2 = json.loads(wbdata.text)
data = wbdata2['data']
for news in data:
title = news['title']
news_url = news['source_url']
news_url = 'https://www.toutiao.com' + news_url
print(title, news_url)
if 'ad_label' in news:
print(news['ad_label'])
continue
download(title,news_url)
next_data = wbdata2['next']
next_max_behot_time = next_data['max_behot_time']
# print("next_max_behot_time:{0}".format(next_max_behot_time))
return next_max_behot_time
if __name__ == '__main__':
refresh = 50
for x in range(0,refresh+1):
print('第{0}次：'.format(x))
if x == 0:
max_behot_time = 0
else:
max_behot_time = next_max_behot_time
#print(next_max_behot_time)
AS,CP = get_ASCP()
url = get_url(max_behot_time,AS,CP)
next_max_behot_time = get_item(url)
四、UC
UC和新浪差不多，没有太复杂的反爬虫，直接解析爬取就好。
from bs4 import BeautifulSoup
from urllib import request
def download(title,url):
req = request.Request(url)
response = request.urlopen(req)
response = response.read().decode('utf-8')
soup = BeautifulSoup(response,'lxml')
tag = soup.find('div',class_='sm-article-content')
if tag == None:
return 0
title = title.replace(':','')
title = title.replace('"','')
title = title.replace('|','')
title = title.replace('/','')
title = title.replace('\\','')
title = title.replace('*','')
title = title.replace('','')
title = title.replace('?','')
with open(r'D:\code\python\spider_news\UC_news\society\\' + title + '.txt','w',encoding='utf-8') as file_object:
file_object.write('\t\t\t\t')
file_object.write(title)
file_object.write('\n')
file_object.write('该新闻地址：')
file_object.write(url)
file_object.write('\n')
file_object.write(tag.get_text())
#print('正在爬取')
if __name__ == '__main__':
for i in range(0,7):
url = 'https://news.uc.cn/c_shehui/'
# headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.91 Safari/537.36",
# "cookie":"sn=3957284397500558579; _uc_pramas=%7B%22fr%22%3A%22pc%22%7D"}
# res = request.Request(url,headers = headers)
res = request.urlopen(url)
req = res.read().decode('utf-8')
soup = BeautifulSoup(req,'lxml')
#print(soup.prettify())
tag = soup.find_all('div',class_ = 'txt-area-title')
#print(tag.name)
for x in tag:
news_url = 'https://news.uc.cn' + x.a.get('href')
print(x.a.string,news_url)
download(x.a.string,news_url)
|