Python爬虫实战之批量下载快手平台视频数据

import json
import requests
import re

data = {
    'operationName': "visionSearchPhoto",
    'query': "query visionSearchPhoto($keyword: String, $pcursor: String, $searchSessionId: String, $page: String, $webPageArea: String) {\n  visionSearchPhoto(keyword: $keyword, pcursor: $pcursor, searchSessionId: $searchSessionId, page: $page, webPageArea: $webPageArea) {\n    result\n    llsid\n    webPageArea\n    feeds {\n      type\n      author {\n        id\n        name\n        following\n        headerUrl\n        headerUrls {\n          cdn\n          url\n          __typename\n        }\n        __typename\n      }\n      tags {\n        type\n        name\n        __typename\n      }\n      photo {\n        id\n        duration\n        caption\n        likeCount\n        realLikeCount\n        coverUrl\n        photoUrl\n        liked\n        timestamp\n        expTag\n        coverUrls {\n          cdn\n          url\n          __typename\n        }\n        photoUrls {\n          cdn\n          url\n          __typename\n        }\n        animatedCoverUrl\n        stereoType\n        videoRatio\n        __typename\n      }\n      canAddComment\n      currentPcursor\n      llsid\n      status\n      __typename\n    }\n    searchSessionId\n    pcursor\n    aladdinBanner {\n      imgUrl\n      link\n      __typename\n    }\n    __typename\n  }\n}\n",
    'variables': {
        'keyword': '张三',
        'pcursor': ' ',
        'page': "search",
        'searchSessionId': "MTRfMjcwOTMyMTQ2XzE2Mjk5ODcyODQ2NTJf5oWi5pGHXzQzMQ"
    }

response = requests.post('https://www.kuaishou.com/graphql', data=data)

headers = {
    # Content-Type（内容类型）的格式有四种(对应data)：分别是
    # 爬虫基础/xml: 把xml作为一个文件来传输
    # multipart/form-data: 用于文件上传
    'content-type': 'application/json',
    # 用户身份标识
    'Cookie': 'kpf=PC_WEB; kpn=KUAISHOU_VISION; clientid=3; did=web_721a784b472981d650bcb8bbc5e9c9c2',
    # 浏览器信息 （伪装成浏览器发送请求）
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
}

# json数据交换格式, 在JSON出现之前, 大家一直用XML来传递数据
# 由于各个语言都支持 JSON ，JSON 又支持各种数据类型，所以JSON常用于我们日常的 HTTP 交互、数据存储等。
# 将python对象编码成Json字符串
data = json.dumps(data)
json_data = requests.post('https://www.kuaishou.com/graphql', headers=headers, data=data).json()

feeds = json_data['data']['visionSearchPhoto']['feeds']
for feed in feeds:
    caption = feed['photo']['caption']
    photoUrl = feed['photo']['photoUrl']
    new_title = re.sub(r'[/\:*?<>/\n] ', '-', caption)

resp = requests.get(photoUrl).content

with open('video\\' + title + '.mp4', mode='wb') as f:
    f.write(resp)
print(title, '爬取成功！！！')

import concurrent.futures
import time

def get_json(url, data):
    response = requests.post(url, headers=headers, data=data).json()
    return response

def change_title(title):
    # windows系统文件命名 不能含有特殊字符...
    # windows文件命名 字符串不能超过 256...
    new_title = re.sub(r'[/\\|:?<>"*\n]', '_', title)
    if len(new_title) > 50:
        new_title = new_title[:10]
    return new_title

def parse(json_data):
    data_list = json_data['data']['visionSearchPhoto']['feeds']
    info_list = []
    for data in data_list:
        # 提取标题
        title = data['photo']['caption']
        new_title = change_title(title)
        url_1 = data['photo']['photoUrl']
        info_list.append([new_title, url_1])
    return info_list

def save(title, url_1):
    resp = requests.get(url_1).content
    with open('video\\' + title + '.mp4', mode='wb') as f:
        f.write(resp)
    print(title, '爬取成功！！！')

def run(url, data):
    """主函数 调动所有的函数"""
    json_data = get_json(url, data)
    info_list = parse(json_data)
    for title, url_1 in info_list:
        save(title, url_1)

if __name__ == '__main__':
    start_time = time.time()
    with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
        for page in range(1, 5):
            url = 'https://www.kuaishou.com/graphql'
            data = {
                'operationName': "visionSearchPhoto",
                'query': "query visionSearchPhoto($keyword: String, $pcursor: String, $searchSessionId: String, $page: String, $webPageArea: String) {\n  visionSearchPhoto(keyword: $keyword, pcursor: $pcursor, searchSessionId: $searchSessionId, page: $page, webPageArea: $webPageArea) {\n    result\n    llsid\n    webPageArea\n    feeds {\n      type\n      author {\n        id\n        name\n        following\n        headerUrl\n        headerUrls {\n          cdn\n          url\n          __typename\n        }\n        __typename\n      }\n      tags {\n        type\n        name\n        __typename\n      }\n      photo {\n        id\n        duration\n        caption\n        likeCount\n        realLikeCount\n        coverUrl\n        photoUrl\n        liked\n        timestamp\n        expTag\n        coverUrls {\n          cdn\n          url\n          __typename\n        }\n        photoUrls {\n          cdn\n          url\n          __typename\n        }\n        animatedCoverUrl\n        stereoType\n        videoRatio\n        __typename\n      }\n      canAddComment\n      currentPcursor\n      llsid\n      status\n      __typename\n    }\n    searchSessionId\n    pcursor\n    aladdinBanner {\n      imgUrl\n      link\n      __typename\n    }\n    __typename\n  }\n}\n",
                'variables': {
                    'keyword': '曹芬',
                    # 'keyword': keyword,
                    'pcursor': str(page),
                    'page': "search",
                    'searchSessionId': "MTRfMjcwOTMyMTQ2XzE2Mjk5ODcyODQ2NTJf5oWi5pGHXzQzMQ"
                }
            }
            data = json.dumps(data)
            executor.submit(run, url, data, )
    print('一共花费了：', time.time()-start_time)