详解python定时简单爬取网页新闻存入数据库并发送邮件

# 爬虫相关模块
import requests
from bs4 import BeautifulSoup
import pymysql

# 发邮件相关模块
import smtplib
from email.mime.text import MIMEText   
from email.header import Header 
import time

# 定时模块
import schedule

# 连接获取html文本
def getHTMLtext(url):
  try:
    headers={
        "user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36",
      } # 浏览器请求头
    r = requests.get(url, headers = headers, timeout = 30) # 获取连接
    r.raise_for_status() # 测试连接是否成功，若失败则报异常
    r.encoding = r.apparent_encoding # 解析编码
    return r.text
  except:
    return ""

# 解析html提取数据
def parseHTML(news, html):
  soup = BeautifulSoup(html, "html.parser") # 获取soup
  for i in soup.find(attrs = {'class' : 'list'}).find_all('li'): # 存放新闻的li标签
    date = i.p.string + '-' + i.h2.string # 日期
    href = i.a['href'] # 链接
    title = i.find('h1').string # 标题
    content = i.find_all('p')[1].string # 梗概
    news.append([date, href, title, content]) # 添加到列表中

# 存入数据库
def toMysql(news):
  conn = pymysql.connect(host = 'localhost', port = 3306, user = 'root', passwd = '数据库密码', db = '数据库名称',charset = 'gbk', connect_timeout = 1000)
  cursor = conn.cursor()
  
  sql = '''
  create table if not exists tb_news(
    日期 date, 
    链接 varchar(400),
    标题 varchar(400),
    梗概 varchar(400))
  '''
  
  cursor.execute(sql) # 建表
  
  for new in news: # 循环存入数据
    sql = 'insert into tb_news(日期, 链接, 标题, 梗概) values(%s, %s, %s, %s)'
    date = new[0]
    href = new[1]
    title = new[2]
    content = new[3]
    cursor.execute(sql, (date, href, title, content))
    
  conn.commit()
  conn.close()

# 发送邮件
def sendMail(news):
  from_addr = '发送邮箱' # 发送邮箱
  password = '16位授权码' # 邮箱授权码
  
  to_addr = '接收邮箱' # 接收邮箱
  
  mailhost = 'smtp.qq.com' # qq邮箱的smtp地址
  qqmail = smtplib.SMTP() # 建立SMTP对象
  qqmail.connect(mailhost, 25) # 25为SMTP常用端口
  qqmail.login(from_addr, password) # 登录邮箱
  
  content = ''
  for new in news: # 拼接邮件内容字符串
    content += '新闻时间：' + new[0] + '\n' + '新闻链接：' + new[1] + '\n' + '新闻标题：' + new[2] + '\n' + '新闻梗概：' + new[3] + '\n'
    content += '======================================================================\n'
    
  # 拼接题目字符串
  subject = time.strftime('%Y-%m-%d %X', time.localtime(time.time())) + '时爬取的北工大首页主要新闻\n'
  
  # 加工邮件message格式
  msg = MIMEText(content, 'plain', 'utf-8')
  msg['subject'] = Header(subject, 'utf-8')
  
  try:
    qqmail.sendmail(from_addr, to_addr, msg.as_string())
    print('发送成功')
  except:
    print('发送失败')
  qqmail.quit()

# 主函数
def main():
  news = []
  url = "http://www.bjut.edu.cn/"
  html = getHTMLtext(url)
	parseHTML(news, html)
	toMysql(news)
  print(news)
	sendMail(news)

main() #测试需要，之后会删除

# 定时执行整个任务
schedule.every().monday.at("08:00").do(main) # 每周一早上八点执行main函数
while True:
  schedule.run_pending()
  time.sleep(1)

schedule.every(5).seconds.do(main)

# 爬虫相关模块
import requests
from bs4 import BeautifulSoup
import pymysql

# 发邮件相关模块
import smtplib
from email.mime.text import MIMEText   
from email.header import Header 
import time

# 定时模块
import schedule

# 连接获取html文本
def getHTMLtext(url):
  try:
    headers={
        "user-agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36",
    } # 浏览器请求头
    r = requests.get(url, headers = headers, timeout = 30) # 获取连接
    r.raise_for_status() # 测试连接是否成功，若失败则报异常
    r.encoding = r.apparent_encoding # 解析编码
    return r.text
  except:
    return ""


# 解析html提取数据
def parseHTML(news, html):
  soup = BeautifulSoup(html, "html.parser") # 获取soup
  for i in soup.find(attrs = {'class' : 'list'}).find_all('li'): # 存放新闻的li标签
    date = i.p.string + '-' + i.h2.string # 日期
    href = i.a['href'] # 链接
    title = i.find('h1').string # 标题
    content = i.find_all('p')[1].string # 梗概
    news.append([date, href, title, content]) # 添加到列表中

# 存入数据库
def toMysql(news):
  conn = pymysql.connect(host = 'localhost', port = 3306, user = 'root', passwd = '数据库密码', db = '数据库名称',charset = 'gbk', connect_timeout = 1000)
  cursor = conn.cursor()
  
  sql = '''
  create table if not exists tb_news(
    日期 date, 
    链接 varchar(400),
    标题 varchar(400),
    梗概 varchar(400))
  '''
  
  cursor.execute(sql) # 建表
  
  for new in news: # 循环存入数据
    sql = 'insert into tb_news(日期, 链接, 标题, 梗概) values(%s, %s, %s, %s)'
    date = new[0]
    href = new[1]
    title = new[2]
    content = new[3]
    cursor.execute(sql, (date, href, title, content))
    
  conn.commit()
  conn.close()

# 发送邮件
def sendMail(news):
  from_addr = '发送邮箱' # 发送邮箱
  password = '16位授权码' # 邮箱授权码
  
  to_addr = '接收邮箱' # 接收邮箱
  
  mailhost = 'smtp.qq.com' # qq邮箱的smtp地址
  qqmail = smtplib.SMTP() # 建立SMTP对象
  qqmail.connect(mailhost, 25) # 25为SMTP常用端口
  qqmail.login(from_addr, password) # 登录邮箱
  
  content = ''
  for new in news: # 拼接邮件内容字符串
    content += '新闻时间：' + new[0] + '\n' + '新闻链接：' + new[1] + '\n' + '新闻标题：' + new[2] + '\n' + '新闻梗概：' + new[3] + '\n'
    content += '======================================================================\n'
    
  # 拼接题目字符串
  subject = time.strftime('%Y-%m-%d %X', time.localtime(time.time())) + '时爬取的北工大首页主要新闻\n'
  
  # 加工邮件message格式
  msg = MIMEText(content, 'plain', 'utf-8')
  msg['subject'] = Header(subject, 'utf-8')
  
  try:
    qqmail.sendmail(from_addr, to_addr, msg.as_string())
    print('发送成功')
  except:
    print('发送失败')
  qqmail.quit()



# 主函数
def main():
  news = []
  url = "http://www.bjut.edu.cn/"
  html = getHTMLtext(url)
  parseHTML(news, html)
  print(news)
  sendMail(news)
  
# 定时执行整个任务
schedule.every().monday.at("08:00").do(main) # 每周一早上八点执行main函数
while True:
  schedule.run_pending()
  time.sleep(1)