Pyhton多线程采集图片方式
作者:三省同学
这篇文章主要介绍了Pyhton多线程采集图片方式,具有很好的参考价值,希望对大家有所帮助,如有错误或未考虑完全的地方,望不吝赐教
前言
需要大量图片做数据采集是,这个时候就需要用到python获取图片,本篇以python多线程获取图片为例。
一、查看页面元素
查看页面源代码。

二、请求url查看
通过F12查看请求url。

tn: resultjson_com logid: 12339447258259285711 ipn: rj ct: 201326592 is: fp: result fr: word: 动漫图片 queryWord: 动漫图片 cl: 2 lm: -1 ie: utf-8 oe: utf-8 adpicid: st: -1 z: ic: hd: latest: copyright: s: se: tab: width: height: face: 0 istype: 2 qc: nc: 1 expermode: nojc: isAsync: pn: 60 rn: 30 gsm: 3c 1669373933133:
代码实现
# -*- coding: utf-8 -*-
import os
import re
import time
from multiprocessing import Pool
import requests
from multiprocessing.dummy import Pool as ThreadPool # 线程池
def get_image(keyword, page_num, save_dir):
# 浏览器伪装
header = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'}
# 请求url
url = 'https://image.baidu.com/search/acjson?'
n = 0;
pn = 1 # pn是从第几张图片开始
page_num = page_num + 1;
for m in range(1, page_num):
# 请求参数
param = {'tn': 'resultjson_com',
'ipn': 'rj',
'ct': 201326592,
'is': '',
'fp': 'result',
'queryWord': keyword,
'cl': 2,
'lm': -1,
'ie': 'utf-8',
'oe': 'utf-8',
'adpicid': '',
'st': -1,
'z': '',
'ic': '',
'hd': 1,
'latest': '',
'copyright': '',
'word': keyword,
's': '',
'se': '',
'tab': '',
'width': '',
'height': '',
'face': 0,
'istype': 2,
'qc': '',
'nc': '1',
'fr': '',
'expermode': '',
'force': '',
'cg': '',
'pn': pn,
'rn': '30',
'gsm': '1e',
'1669373933133': ' '
}
request = requests.get(url=url, headers=header, params=param)
if request.status_code == 200:
print('success.')
request.encoding = 'utf-8'
html = request.text
image_url_list = re.findall('"thumbURL":"(.*?)",', html, re.S)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
for image_url in image_url_list:
image_data = requests.get(url=image_url, headers=header).content
# with open(os.path.join(save_dir, "{}_{:06d}.jpg".format("1", n)), 'wb') as fp:
# fp.write(image_data)
pool.apply_async(download, args=(n, image_data, save_dir), error_callback=func.err_call_back)
n = n + 1
pn += 29
class Func(object):
def __init__(self):
# 利用匿名函数模拟一个不可序列化象
# 更常见的错误写法是,在这里初始化一个数据库的长链接
self.num = lambda: None
def work(self, num=None):
self.num = num
return self.num
@staticmethod
def call_back(res):
print('Hello,World! {res}')
@staticmethod
def err_call_back(err):
print('出错啦:[{}]'.format(err))
def download(n, image_data, save_dir):
# time.sleep(1)
fp = open(os.path.join(save_dir, "{}_{:06d}.jpg".format("1", n)), 'wb')
fp.write(image_data)
fp.close()
if __name__ == '__main__':
func = Func()
keyword = '动漫图片'
save_dir = keyword
page_num = int(input("页数:"))
# 线程池中线程数
pool = Pool(10)
# pool = ThreadPool(5)
# i = 0;
# while i < page_num:
get_image(keyword, page_num, save_dir)
# i = i + 1
print('完成')
总结
以上为个人经验,希望能给大家一个参考,也希望大家多多支持脚本之家。
