Python爬虫原理与基本请求库urllib详解

#无论哪种方法，先导包
import urllib.request

'''
urllib.request.urlopen(url, data, timeout) 发送request请求（3个参数不需都写，但必须有url）
url：目标网址
data：访问url时发送的数据包，默认为null
timeout：等待时长
'''
response = urllib.request.urlopen('http://www.baidu.com/')
# decode()：解码，将字节流格式数据以相应的形式转化为字符串
print(response.read().decode('utf-8'))

'''
urllib.request.urlretrieve(url,filename) 直接将目标网址的网页信息保存到本地
filename：想保存到的路径文件名
然后用urllib.request.urlcleanup()清除缓存
'''
filename = urllib.request.urlretrieve('http://www.baidu.com/',filename='baidu.html')
urllib.request.urlcleanup()

# 定义关键词
keyword = input('输入你想查询的关键词:')
# 处理中文，进行编码，转成字节流格式
keyword = urllib.request.quote(keyword)
# url重构
url = 'http://www.baidu.com/s?wd='+keyword
response = urllib.request.urlopen(url).read()
# 保存到本地
f = open('baidu_search.html','wb') #wb存储字节流数据，所以上述response不需要用decode解码
f.write(response)
f.close()

import urllib.request
import urllib.parse #上传用户名和密码，需要先用parse模块进行编码处理
url = 'https://www.iqianyue.com/mypost/'
# urlencode():将上传的数据进行编码处理
# encode():将字符串转化为相应编码的字节流数据
postdata = urllib.parse.urlencode({
    'name':'vicky',
    'pass':'12345'
}).encode('utf-8')
response = urllib.request.urlopen(url,data=postdata).read()
# 保存网页
f = open('post.html','wb')
f.write(response)
f.close()

import urllib.request
import urllib.parse
# 主页面：'http://blog.chinaunix.net/'
# 登录页面：'http://account.chinaunix.net/login?url=http%3a%2f%2fblog.chinaunix.net'
# 登录跳转页面：'http://account.chinaunix.net/login/sso?url=http%3A%2F%2Fblog.chinaunix.net'
url = 'http://account.chinaunix.net/login/sso?url=http%3A%2F%2Fblog.chinaunix.net'
postdata = urllib.parse.urlencode({
    'username':'***',
    'password':'******'
}).encode('utf-8')
response = urllib.request.urlopen(url,postdata).read()
f = open('chinaunix.html','wb')
f.write(response)
f.close()

import urllib.request
import urllib.parse
import http.cookiejar
url = 'http://account.chinaunix.net/login/sso?url=http%3A%2F%2Fblog.chinaunix.net'
postdata = urllib.parse.urlencode({
    'username':'***',
    'password':'***'
}).encode('utf-8')
response = urllib.request.urlopen(url,postdata).read()
# 创建cookiejar对象
cjar = http.cookiejar.CookieJar()
# 创建cookie处理器，并以此为参数构建opener对象
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cjar))
# 加载为全局默认的opener
urllib.request.install_opener(opener)
f = open('chinaunix_2.html','wb')
f.write(response)
f.close()

import urllib.request
url = 'http://t.dianping.com/hangzhou'
# 构建headers
h = {
    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36'
}
# 构建Request请求对象
request = urllib.request.Request(url=url, headers=h)
response = urllib.request.urlopen(request).read()
# 保存文件
f = open('headers.html','wb')
f.write(response)
f.close()

import urllib.request
def use_Proxy(proxy_addr,url):
    # ProxyHandler()设置对应的代理服务器信息
    proxy = urllib.request.ProxyHandler({'http':proxy_addr})
    # build_opener()创建一个自定义的opener对象
    opener = urllib.request.build_opener(proxy)
    # 将opener加载为全局使用的opener
    urllib.request.install_opener(opener)
    response = urllib.request.urlopen(url).read().decode('utf-8')
    return response
proxy_addr = '183.146.213.157:80'  # IP地址:端口号  自行百度“代理IP”
url = 'http://www.taobao.com'
print(len(use_Proxy(proxy_addr,url)))  #随便打印一个长度，看看这个代理IP能否使用

import urllib.request
import urllib.error
try:
    response = urllib.request.urlopen('http://www.ijinqian.com')
except urllib.error.URLError as e:
    print('发生异常--->'+str(e))