Jan-06-2021, 02:17 PM
Can show a example and i use my code
Here use concurrent.futures
Here use concurrent.futures
import requests from bs4 import BeautifulSoup from os import path import concurrent.futures def read_url(url): headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36'} response = requests.get(url, headers=headers) soup = BeautifulSoup(response.content, 'lxml') img_all = soup.select('div.thread_image_box > a') return img_all def img_multi(img_link): print(f'Download --> {path.basename(img_link)}') with open(path.basename(img_link), 'wb') as f_out: f_out.write(requests.get(img_link).content) if __name__ == '__main__': url = 'https://archive.4plebs.org/hr/thread/2866456/' img_all = read_url(url) # ThreadPoolExecutor | ProcessPoolExecutor with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: for img in img_all: img_link = img.get('href') executor.submit(img_multi, img_link)