Jan-12-2021, 04:10 PM
Man, that's awesome. I love the concurrent.futures part! I went ahead though and bastardized your code too to ask me for a folder name and then change the name a titch. this is what I ended up with.
import requests
from bs4 import BeautifulSoup
from os import path
import os
import concurrent.futures
url = input("Website:")
folder = input("Folder:")
def read_url(url):
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36'}
response = requests.get(url, headers=headers)
soup = BeautifulSoup(response.content, 'lxml')
img_all = soup.select('div.thread_image_box > a')
return img_all
def img_multi(img_link):
name = folder + ' - ' + path.basename(img_link)
print(f'Download --> ',name)
dlpath = os.path.join(folder, name)
with open(dlpath, 'wb') as f_out:
f_out.write(requests.get(img_link).content)
if __name__ == '__main__':
img_all = read_url(url)
if not os.path.isdir(folder):
os.makedirs(folder)
# ThreadPoolExecutor | ProcessPoolExecutor
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
for img in img_all:
img_link = img.get('href')
executor.submit(img_multi, img_link)took me far longer than you probably could image to figure out how to actually get it to ask for a name and then use it. But, it's there! Thanks for sharing your code!