This code takes images from a page and prints them:
import os
import requests
from bs4 import BeautifulSoup
downloadDirectory = "downloaded"
baseUrl = "http://pythonscraping.com"
def getAbsoluteURL(baseUrl, source):
if source.startswith("http://www."):
url = "http://"+source[11:]
elif source.startswith("http://"):
url = source
elif source.startswith("www."):
url = source[4:]
url = "http://"+source
else:
url = baseUrl+"/"+source
if baseUrl not in url:
return None
return url
def getDownloadPath(baseUrl, absoluteUrl, downloadDirectory):
path = absoluteUrl.replace("www.", "")
path = path.replace(baseUrl, "")
path = downloadDirectory+path
directory = os.path.dirname(path)
if not os.path.exists(directory):
os.makedirs(directory)
return path
html = requests.get("http://www.pythonscraping.com")
bsObj = BeautifulSoup(html.content, 'html.parser')
downloadList = bsObj.find_all(src=True)
now I want to download these images to the directory downloaded so I added this piece of code:
with open(fileUrl, getDownloadPath(baseUrl, fileUrl, downloadDirectory)) as out_file:
out_file.write(fileUrl.content)
but I get this:
Error:
http://pythonscraping.com/misc/jquery.js?v=1.4.4
http://pythonscraping.com/misc/jquery.once.js?v=1.2
http://pythonscraping.com/misc/drupal.js?pa2nir
http://pythonscraping.com/sites/all/themes/skeletontheme/js/jquery.mobilemenu.js
?pa2nir
http://pythonscraping.com/sites/all/modules/google_analytics/googleanalytics.js?
pa2nir
http://pythonscraping.com/sites/default/files/lrg_0.jpg
http://pythonscraping.com/img/lrg%20(1).jpg
Traceback (most recent call last):
File "C:\Python36\kodovi\crawler3.py", line 76, in <module>
with open(fileUrl, getDownloadPath(baseUrl, fileUrl, downloadDirectory)) as
out_file:
ValueError: invalid mode: 'downloaded/img/lrg%20(1).jpg'
p.s. to add that execution of this code opened folder downloaded and empty folder img within it.