Python: Requests 基本使用、代理、超时、认证、异常、上传文件

发表 admin at 2022年7月27日

类别

Python

标签

1 爬虫介绍
2 requests模块介绍
3 携带get参数
4 携带请求头
5 携带cookie
6 post请求
7 post请求携带数据
8 request.session
9 response属性
10 编码问题
11 获取二进制数据
11 解析json
13高级用法之 Cert Verification
14 超时，认证，异常，上传文件
案例：爬取某视频网站

1 爬虫介绍

# 所有的软件，90%以上，cs,bs,主流都是用http协议通信，模拟发送http请求---》服务端把数据返回(html,xml,json)---->数据的清洗（re，bs4）---->入库（文件，mysql，redis，es，mongo）
	-mysql：tcp自定定制的协议
    -redis：tcp自定定制的协议
    -docker：http协议，符合resful规范
    -es：http协议，符合resful规范

# 爬虫的本质：
	1 模拟发送http请求(requests，requests-hmtl,selenium)
    2 数据清洗反扒（re，bs4，lxml：css选择器，xpath选择器）
    3 增加并发量（线程，进程，协程）---》scrapy
    4 入库
    
# 最大的爬虫：百度，谷歌
	-百度一刻不停的在互联网中爬取网页----》把网页存到百度的数据库
    	(基本上所有公司都希望被百度爬到，并且权重高，显示在最前面)----（seo，sem）
    -当我们在搜索内容时，输入了关键字----》全文检索---》现在在搜索结果中


# 爬虫有爬虫协议
	-我们做爬虫要遵循爬虫协议：这个网站哪些让我们爬，哪些不让
    -robots.txt


# 编码，混淆，加密的区别
# js逆向，安卓反编译，动态调试（汇编）

2 requests模块介绍

# 模拟发送http请求，requests模块
# urllib内置模块，可以发送http请求，但是api使用复杂
# 介绍：使用requests可以模拟浏览器的请求，比起之前用到的urllib，requests模块的api更加便捷（本质就是封装了urllib3）

# 注意：requests库发送请求将网页内容下载下来以后，并不会执行js代码，这需要我们自己分析目标站点然后发起新的request请求

# 安装：pip3 install requests
# 只能发送请求，不能解析html，后来作者又写了一个模块：requests-hmtl模块：兼具发送请求和解析
	requests+lxml=requests-hmtl
    
    
# 不仅仅用来做爬虫，重点：后端跟另一个服务交互，也需要使用它
	-假设公司有：短链系统    把很长的url链接生成短的url链接
    	-原理：申请很短的域名：https://sourl.cn/wBdfzc
        	id    code        url
            1     wBdfzc     https://image.baidu.com/search/index?
            
    -其它系统生成的链接很长---》转短---》调用短链系统的接口

import requests

# res=requests.get('https://www.cnblogs.com/')  # res是http响应封装成了对象，响应中得所有东西，都在这个对象中
# print(res.text)  # 响应体的字符串

3 携带get参数

# 2 请求地址中带参数
# 方式一：直接使用
# res=requests.get('https://www.cnblogs.com/?name=lqz&age=19')
# print(res.text)  # 响应体的字符串
# 方式二：使用params参数
# res=requests.get('https://www.cnblogs.com/',params={'name':'lqz','age':19})
# print(res.text)  # 响应体的字符串
# 小注意：如果中文涉及到 url的编码和解码
# https://www.cnblogs.com/?name=%E6%9D%8E%E6%B8%85%E7%85%A7&age=19
#                              %E6%9D%8E%E6%B8%85%E7%85%A7
from urllib import parse

#
# res=parse.quote('李清照')
# print(res)
#
# res1=parse.unquote('%E6%9D%8E%E6%B8%85%E7%85%A7')
# res1=parse.unquote('http://www.aa7a.cn/user.php?&ref=http%3A%2F%2Fwww.aa7a.cn%2Fuser.php%3Fact%3Dlogout')
# print(res1)

4 携带请求头

# 3 携带请求头
爬不到---》做了反扒————》最初级的反扒----》先在请求头中得user-agent
# 常见请求头：
# user-agent：客户端类型
Referer: https://www.lagou.com/gongsi/  上一次访问的地址
# 图片防盗链
反扒：除了首页外，上一个地址不是我们自己的地址，就禁掉


# 面试题：讲讲你知道的http的请求头？ x-forword-for
header = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36',
    'Cookie':...,
}
res=requests.get('https://dig.chouti.com/',headers=header)
print(res.text)
# {"msg":"你已经推荐过了","code":400,"errorType":0,"success":false}

5 携带cookie

# 4 携带cookie---》登录信息---》携带着说明登录了---》能干登录的事
## 方式一：带在请求头中
res = requests.post('https://dig.chouti.com/link/vote',
                    headers=header,
                    data={
                        'linkId': '35811284'
                    })
print(res.text)

# 方式二：使用cookie参数:之前登录成功了，就有cookie，cookie是CookieJar的对象，直接传
header = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36',
}
# requests.get()
res = requests.post('https://dig.chouti.com/link/vote',
                    headers=header,
                    # Dict or CookieJar
                    cookies={'Cookie':...},
                    data={
                        'linkId': '35811284'
                    })

print(res.text)
# {"msg":"你已经推荐过了","code":400,"errorType":0,"success":false}

6 post请求

# 5 发送post请求---》登录接口
# from-data,urlencoded(默认),json
res = requests.post('http://www.aa7a.cn/user.php', data={
    'username': '616564099@qq.com',
    'password': 'lqz123',
    'captcha': 'aaaa',
    'remember': 1,
    'ref': 'http://www.aa7a.cn/user.php?act=logout',
    'act': ' act_login',
})
print(res.text)
print(res.cookies)  # 登录成功的cookie，cookieJar对象--》字典
# 登录成功
res1=requests.get('http://www.aa7a.cn/',cookies=res.cookies)
print('616564099@qq.com' in res1.text)

7 post请求携带数据

# 6 post请求编码是json格式
# res=requests.post('xxx',json={})

8 request.session

# 7 request.session的使用，整个过程中自动维护cookie

session=requests.session()
# 使用session发送请求
session.post('http://www.aa7a.cn/user.php', data={
    'username': '616564099@qq.com',
    'password': 'lqz123',
    'captcha': 'aaaa',
    'remember': 1,
    'ref': 'http://www.aa7a.cn/user.php?act=logout',
    'act': ' act_login',
})
res1=session.get('http://www.aa7a.cn/')
print('616564099@qq.com' in res1.text)
# True

9 response属性

# 8 repsonse对象的属性和方法--》把http的响应封装成了response
respone=requests.get('https://www.cnblogs.com/')
respone1=requests.get('http://www.autohome.com/news')
print(respone.text)   # 响应体的字符串
print(respone.content) # 响应体二进制数据
print(respone.status_code) #响应状态码
print(respone.headers)# 响应头
print(respone.cookies) #响应的cookie
print(respone.cookies.get_dict()) #cookie转成dict
print(respone.cookies.items())  # cookie拿出key和value
print(respone.url)         # 请求的地址
print(respone.history)     # 列表，有重定向，里面放了重定向之前的地址
print(respone.encoding)   # 响应编码格式

# 后期咱们下载图片，视频，需要使用它
respone.iter_content()
res=requests.get('https://video.pearvideo.com/mp4/adshort/20220427/cont-1760318-15870165_adpkg-ad_hd.mp4')
with open('致命诱惑3.mp4','wb') as f:
    # f.write(res.content)
    for line in res.iter_content(chunk_size=1024): # 按1024字节写
        f.write(line)

10 编码问题

# 9 编码问题---》大部分网站都是utf-8编码，老网站中文编码使用gbk，gb2312
respone = requests.get('http://www.autohome.com/news')
# respone.encoding='gbk'
print(respone.text)  # 默认使用utf-8可能会导致中文乱码，这时可以加上上面一句

11 获取二进制数据

# 10 获取二进制数据
response.content
response.iter_content(chunk_size=1024)
res=requests.get('https://gd-hbimg.huaban.com/e1abf47cecfe5848afc2a4a8fd2e0df1c272637f2825b-e3lVMF_fw658')
with open('美女.png','wb') as f:
    f.write(res.content)

11 解析json

# 获取返回值
    res = requests.post('http://www.kfc.com.cn/kfccda/ashx/GetStoreList.ashx?op=keyword', data={
        'cname': '',
        'pid': '',
        'keyword': '北京',
        'pageIndex': 1,
        'pageSize': 10,
    })
    print(res.text)
    # ...
    print(type(res.text))
    # <class 'str'>  字符串格式

# 转成json格式
    import json
    dic_res = json.loads(res.text)
    print(type(dic_res))
    # <class 'dict'>
    print(dic_res['Table1'][0]['storeName'])
    # 京通新城

# 简便方法
    res = requests.post('http://www.kfc.com.cn/kfccda/ashx/GetStoreList.ashx?op=keyword', data={
        'cname': '',
        'pid': '',
        'keyword': '北京',
        'pageIndex': 1,
        'pageSize': 10,

    })
    print(type(res.json()))
    # <class 'dict'>

13高级用法之 Cert Verification

# 13 高级用法之证书（了解）----》http请求---》https请求
什么是https=http+ssl

import requests
# 不验证证书，直接访问
respone=requests.get('https://www.12306.cn',verify=False)
# 发送请求手动携带证书
respone=requests.get('https://www.12306.cn',
                     cert=('/path/server.crt',  # 服务器证书
                           '/path/key'))        # 服务器密钥

14 超时，认证，异常，上传文件

# 15 超时设置
import requests
respone=requests.get('https://www.baidu.com',timeout=0.0001)


# 16 异常处理
from requests.exceptions import *
try:
    r=requests.get('http://www.baidu.com',timeout=0.00001)
# except ReadTimeout:
#     print('===:')
# except ConnectionError: #网络不通
#     print('-----')
# except Timeout:
#     print('aaaaa')
except Exception:
    print('x')


# 17 上传文件
import requests
files={'file':open('a.jpg','rb')}
respone=requests.post('http://httpbin.org/post',files=files)
print(respone.status_code)

案例：爬取某视频网站

# https://www.pearvideo.com/category_loading.jsp?reqType=5&categoryId=5&start=0
# 好看视频---》json

import requests
import re

res = requests.get('https://www.pearvideo.com/category_loading.jsp?reqType=5&categoryId=5&start=0')
# print(res.text)

video_list = re.findall('<a href="(.*?)" class="vervideo-lilink actplay">', res.text)
print(video_list)
# https://www.pearvideo.com/video_1768482
for video in video_list:
    video_id = video.split('_')[-1]
    video_url = 'https://www.pearvideo.com/' + video
    # 请求头中携带所需参数
    header = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36',
        'Referer': video_url
    }
    res1 = requests.get('https://www.pearvideo.com/videoStatus.jsp?contId=%s&mrd=0.5602821872545047' % video_id,
                        headers=header
                        ).json()
    # 获取视频地址
    # print(res1['videoInfo']['videos']['srcUrl'])
    mp4_url = res1['videoInfo']['videos']['srcUrl']
    real_mp4_url = mp4_url.replace(mp4_url.split('/')[-1].split('-')[0], 'cont-%s' % video_id)
    print(real_mp4_url)
    # 下载视频
    res2 = requests.get(real_mp4_url)
    with open('video/%s.mp4' % video_id, 'wb') as f:
        for line in res2.iter_content(1024):
            f.write(line)

# 直接发送请求，拿不到视频，它是发送了ajax请求获取了视频，但是需要携带referer
# res=requests.get('https://www.pearvideo.com/video_1768482')
# print(res.text)


# https://video.pearvideo.com/mp4/third/20220729/     1659324669265     -11320310-183708-hd.mp4   # 不能播
# https://video.pearvideo.com/mp4/third/20220729/     cont-1768482      -11320310-183708-hd.mp4    #能播

# url='https://video.pearvideo.com/mp4/third/20220729/   1659324669265    -11320310-183708-hd.mp4'


# 其他拓展
    -关于全站爬取：更好分类id和起始爬取的数字即可
    -同步爬取，速度一般，加入线程(线程池)，提高爬取速度---》作业（线程池）
    -封ip问题---》代理池了（使用代理池）
    -视频处理（截取视频，拼接视频）---》ffmpeg软件---》通过命令调用软件
    	python操作软件：subprocess模块 执行ffmpeg的命令完成视频操作
       	python模块操作opencv（c写的，编译后，使用python调用），实现非常高级的功能
        文件操作给视频加头去尾部

二、本文目录：

一、post方法简单使用

1、带数据的post

2、带header的post

3、带json的post

4、带参数的post

5、普通文件上传

6、定制化文件上传

7、多文件上传

Python Requests post() Method

Example

Make a POST request to a web page, and return the response text:


import requests

url = 'https://www.w3schools.com/python/demopage.php'
myobj = {'somekey': 'somevalue'}
x = requests.post(url, json = myobj)
print(x.text)

Syntax

requests.post(url, data={key: value}, json={key: value}, args)

args means zero or more of the named arguments in the parameter table below. Example:

requests.post(url, data = myobj, timeout=2.50)

Parameter Values

Parameter		Description
url		Required. The url of the request
data		Optional. A dictionary, list of tuples, bytes or a file object to send to the specified url
json		Optional. A JSON object to send to the specified url
files		Optional. A dictionary of files to send to the specified url
allow_redirects		Optional. A Boolean to enable/disable redirection. Default `True` (allowing redirects)
auth		Optional. A tuple to enable a certain HTTP authentication. Default `None`
cert		Optional. A String or Tuple specifying a cert file or key. Default `None`
cookies		Optional. A dictionary of cookies to send to the specified url. Default `None`
headers		Optional. A dictionary of HTTP headers to send to the specified url. Default `None`
proxies		Optional. A dictionary of the protocol to the proxy url. Default `None`
stream		Optional. A Boolean indication if the response should be immediately downloaded (False) or streamed (True). Default `False`
timeout		Optional. A number, or a tuple, indicating how many seconds to wait for the client to make a connection and/or send a response. Default `None` which means the request will continue until the connection is closed
verify		Optional. A Boolean or a String indication to verify the servers TLS certificate or not. Default `True`

一、post方法简单使用：1、带数据的post：

# -*- coding:utf-8 -*-
import requests
import json

host = "http://httpbin.org/"
endpoint = "post"
url = ''.join([host,endpoint])
data = {'key1':'value1','key2':'value2'}

r = requests.post(url,data=data)
#response = r.json()
print (r.text)

输出：

{
  "args": {}, 
  "data": "", 
  "files": {}, 
  "form": {
    "key1": "value1", 
    "key2": "value2"
  }, 
  "headers": {
    "Accept": "*/*", 
    "Accept-Encoding": "gzip, deflate", 
    "Connection": "close", 
    "Content-Length": "23", 
    "Content-Type": "application/x-www-form-urlencoded", 
    "Host": "httpbin.org", 
    "User-Agent": "python-requests/2.18.1"
  }, 
  "json": null, 
  "origin": "183.14.133.88", 
  "url": "http://httpbin.org/post"
}

2、带header的post：

# -*- coding:utf-8 -*-
import requests
import json

host = "http://httpbin.org/"
endpoint = "post"

url = ''.join([host,endpoint])
headers = {"User-Agent":"test request headers"}

# r = requests.post(url)
r = requests.post(url,headers=headers)
#response = r.json()

输出：

{
  "args": {}, 
  "data": "", 
  "files": {}, 
  "form": {}, 
  "headers": {
    "Accept": "*/*", 
    "Accept-Encoding": "gzip, deflate", 
    "Connection": "close", 
    "Content-Length": "0", 
    "Host": "httpbin.org", 
    "User-Agent": "test request headers"
  }, 
  "json": null, 
  "origin": "183.14.133.88", 
  "url": "http://httpbin.org/post"
}

3、带json的post：

# -*- coding:utf-8 -*-
import requests
import json

host = "http://httpbin.org/"
endpoint = "post"

url = ''.join([host,endpoint])
data = {
    "sites": [
                { "name":"test" , "url":"www.test.com" },
                { "name":"google" , "url":"www.google.com" },
                { "name":"weibo" , "url":"www.weibo.com" }
    ]
}

r = requests.post(url,json=data)
# r = requests.post(url,data=json.dumps(data))
response = r.json()

输出：

{
  "args": {}, 
  "data": "{\"sites\": [{\"url\": \"www.test.com\", \"name\": \"test\"}, {\"url\": \"www.google.com\", \"name\": \"google\"}, {\"url\": \"www.weibo.com\", \"name\": \"weibo\"}]}", 
  "files": {}, 
  "form": {}, 
  "headers": {
    "Accept": "*/*", 
    "Accept-Encoding": "gzip, deflate", 
    "Connection": "close", 
    "Content-Length": "140", 
    "Content-Type": "application/json", 
    "Host": "httpbin.org", 
    "User-Agent": "python-requests/2.18.1"
  }, 
  "json": {
    "sites": [
      {
        "name": "test", 
        "url": "www.test.com"
      }, 
      {
        "name": "google", 
        "url": "www.google.com"
      }, 
      {
        "name": "weibo", 
        "url": "www.weibo.com"
      }
    ]
  }, 
  "origin": "183.14.133.88", 
  "url": "http://httpbin.org/post"
}

4、带参数的post：

# -*- coding:utf-8 -*-
import requests
import json

host = "http://httpbin.org/"
endpoint = "post"

url = ''.join([host,endpoint])
params = {'key1':'params1','key2':'params2'}

# r = requests.post(url)
r = requests.post(url,params=params)
#response = r.json()
print (r.text)

输出：

{
  "args": {
    "key1": "params1", 
    "key2": "params2"
  }, 
  "data": "", 
  "files": {}, 
  "form": {}, 
  "headers": {
    "Accept": "*/*", 
    "Accept-Encoding": "gzip, deflate", 
    "Connection": "close", 
    "Content-Length": "0", 
    "Host": "httpbin.org", 
    "User-Agent": "python-requests/2.18.1"
  }, 
  "json": null, 
  "origin": "183.14.133.88", 
  "url": "http://httpbin.org/post?key2=params2&key1=params1"
}

5、普通文件上传：

# -*- coding:utf-8 -*-
import requests
import json

host = "http://httpbin.org/"
endpoint = "post"

url = ''.join([host,endpoint])
#普通上传
files = {
            'file':open('test.txt','rb')
        }

r = requests.post(url,files=files)
print (r.text)

输出：

{
  "args": {}, 
  "data": "", 
  "files": {
    "file": "hello world!\n"
  }, 
  "form": {}, 
  "headers": {
    "Accept": "*/*", 
    "Accept-Encoding": "gzip, deflate", 
    "Connection": "close", 
    "Content-Length": "157", 
    "Content-Type": "multipart/form-data; boundary=392865f79bf6431f8a53c9d56c62571e", 
    "Host": "httpbin.org", 
    "User-Agent": "python-requests/2.18.1"
  }, 
  "json": null, 
  "origin": "183.14.133.88", 
  "url": "http://httpbin.org/post"
}

6、定制化文件上传：

# -*- coding:utf-8 -*-
import requests
import json

host = "http://httpbin.org/"
endpoint = "post"

url = ''.join([host,endpoint])
#自定义文件名，文件类型、请求头
files = {
        'file':('test.png',open('test.png','rb'),'image/png')
}

r = requests.post(url,files=files)
print (r.text)heman793

输出比较在，就不帖了。

7、多文件上传：

# -*- coding:utf-8 -*-
import requests
import json

host = "http://httpbin.org/"
endpoint = "post"

url = ''.join([host,endpoint])
#多文件上传
files = [
    ('file1',('test.txt',open('test.txt', 'rb'))),
    ('file2', ('test.png', open('test.png', 'rb')))
    ]

r = requests.post(url,files=files)
print (r.text)

输出上，太多内容，不帖了。

8、流式上传：

# -*- coding:utf-8 -*-
import requests
import json

host = "http://httpbin.org/"
endpoint = "post"

url = ''.join([host,endpoint])

#流式上传
with open( 'test.txt' ) as f:
    r = requests.post(url,data = f)

print (r.text)

输出：

{
  "args": {}, 
  "data": "hello world!\n", 
  "files": {}, 
  "form": {}, 
  "headers": {
    "Accept": "*/*", 
    "Accept-Encoding": "gzip, deflate", 
    "Connection": "close", 
    "Content-Length": "13", 
    "Host": "httpbin.org", 
    "User-Agent": "python-requests/2.18.1"
  }, 
  "json": null, 
  "origin": "183.14.133.88", 
  "url": "http://httpbin.org/post"
}