爬虫学习-第六篇

首先感谢崔庆才博主的文章，以下信息都是在其文章与《python3网络爬虫开发实战》书中摘抄出的https://cuiqingcai.com

"""
Version: 0.1
Author: freshbin
Date: 2019年8月23日
"""

print("=================================使用requests start================================================")

# import requests
#
# r = requests.get('https://www.baidu.com')
# print(type(r))
# print(r.status_code)
# print(type(r.text))
# print(r.text)
# print(r.cookies)

'''
r = request.post('http://httpbin.org/post')
r = request.put('http://httpbin.org/put')
r = request.delete('http://httpbin.org/delete')
r = request.head('http://httpbin.org/get')
r = request.options('http://httpbin.org/get')
'''

# GET请求

# 基本实例
# import requests
#
# data = {
#     'name': 'germey',
#     'age': 22
# }
#
# r = requests.get('http://httpbin.org/get', params=data)
# print(r.text)
# print(r.json())

# 抓取网页
# import requests
# import re
#
# headers = {
#     'User-Agent': 'Mozilla/5.0 (Machintosh; Intel Mac Os X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'
# }
# r = requests.get('https://www.zhihu.com/explore', headers=headers)
# pattern = re.compile('explore-feed.*?question_link.*?>(.*?)</a>', re.S)
# titles = re.findall(pattern, r.text)
# print(titles)
# print(r.text)

# 抓取二进制数据
# import requests
#
# r = requests.get('https://github.com/favicon.ico')
# with open('favicon.ico', 'wb') as f:
#     f.write(r.content)


# POST请求
# import requests
# data = {'name': 'germey', 'age': 22}
# r = requests.post('http://httpbin.org/post', data=data)
# print(r.text)

# 响应
# import requests
#
# r = requests.get('http://www.jianshu.com')
# print(type(r.status_code), r.status_code)
# print(type(r.headers), r.headers)
# print(type(r.cookies), r.cookies)
# print(type(r.url), r.url)
# print(type(r.history), r.history)


# 高级用法
# 1、文件上传
# import requests
#
# files = {'file': open('favicon.ico', 'rb')}
# r = requests.post('http://httpbin.org/post', files=files)
# print(r.text)

# 2、Cookies
# import requests
#
# r = requests.get('http://www.baidu.com')
# print(r.cookies)
#
# for key, value in r.cookies.items():
#     print(key + '=' + value)

# 使用cookie登录
# import requests

# headers = {
#     'Cookie': '_zap=bacc5593-9845-4048-aa82-d0d91f8ff78d; d_c0="ALBmEoNTLw-PTrB3LBOoC2Fo2ZhuDYkGoP4=|1553654046"; _xsrf=8KJLfGg4Cq7MJxtm5Arw52MtQEStemEh; __gads=ID=15b07f1d788bf348:T=1555310933:S=ALNI_MYYhc8Axzu8xASgP9Ms4Wdfkh_9WA; z_c0="2|1:0|10:1557046119|4:z_c0|92:Mi4xc09WSUFnQUFBQUFBc0dZU2cxTXZEeVlBQUFCZ0FsVk5aX0c3WFFEcmFGYkNOUTZMUVZDNDE2VGpnRUczZ2UxNzhn|c54e5d7c48dc3a80dccb468165a5568c13dac71f733eb96a881967b5bf00d7b6"; __utmv=51854390.100--|2=registration_date=20151114=1^3=entry_date=20151114=1; tshl=; __utma=51854390.1077158580.1553654412.1563969553.1564652550.6; __utmz=51854390.1564652550.6.6.utmcsr=zhihu.com|utmccn=(referral)|utmcmd=referral|utmcct=/; tst=h; q_c1=d03071cc8a9947198c6f0984f188426a|1566359825000|1553654438000; tgw_l7_route=116a747939468d99065d12a386ab1c5f',
#     'Host': 'www.zhihu.com',
#     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
# }
# r = requests.get('https://www.zhihu.com', headers=headers)
# print(r.text)

# 3、会话维持
# import requests
#
# s = requests.session()
# s.get('http://httpbin.org/cookies/set/number/123456789')
# r = s.get('http://httpbin.org/cookies')
# print(r.text)

# 4、SSL证书验证
# import requests
#
# response = requests.get('https://www.12306.cn', verify=False) # 12306证书已经被官方CA机构信任了，所以访问不会被提示风险
# print(response.status_code)

# 5、代理设置
# import requests
#
# proxies = {
#     'http': 'http://10.10.1.10:3128',
#     'https': 'http://10.10.1.10:1080'
# }
#
# requests.get('http://www.taobao.com', proxies=proxies)


# 6、超时设置
# import requests

# r = requests.get('https://www.taobao.com', timeout = 1)

# 请求分为两个阶段，即连接和读取 timeout是这两个的总和，也可以分别指定
# r = requests.get('https://www.taobao.com', timeout=(5, 10))

# 如果想永久等待，那么timeout设为None或者不设置timeout参数

# print(r.status_code)

# 7、身份验证
# import requests
# from requests.auth import HTTPBasicAuth
#
# r = requests.get('http://localhost:5000', auth=HTTPBasicAuth('username', 'password'))
# r = requests.get('http://localhost:5000', auth=('username', 'password'))

# print(r.status_code)

# Oauth认证 需要暗转 oauth包， pip install requests_oauthlib 详细功能见：http://requests-oauthlib.readthedocs.org/

# Prepared Request
# from requests import Request, Session
# 
# url = 'http://httpbin.org/post'
# data = {
#     'name': 'germey'
# }
# headers = {
#     'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'
# }
# s = Session()
# req = Request('POST', url, data=data, headers=headers)
# prepped = s.prepare_request(req)
# r = s.send(prepped)
# print(r.text)

# 更多用法见官方文档：http://docs.python-request.org/
print("=================================使用requests end================================================")