Aug-17-2021, 02:48 PM
Hello,
please help python experts. I want to iterate through multiple pages for products info but when loop gets to the second page of products it throws me 'NoneType' object has no attribute 'find' error. Everything looks find to me though.
please help python experts. I want to iterate through multiple pages for products info but when loop gets to the second page of products it throws me 'NoneType' object has no attribute 'find' error. Everything looks find to me though.
from urllib.request import urlopen as uReq from urllib.request import Request import requests from bs4 import BeautifulSoup as soup
pagenumbers = ["1", "2", "3", "4", "5", "6", "7", "8", "9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30"] url = "https://www.costway.ca" first_r = requests.get(url) first_soup = soup(first_r.content,'lxml') allpage_urls = first_soup.find('div',{'class':'category-content shop-by-catagory'}).find_all('a') #print(allpage_urls) for link in allpage_urls: originalurl = "https://www.costway.ca" + link['href'] print(originalurl) for page in pagenumbers: Cat_page_url = originalurl+"?p="+page print(Cat_page_url) third_r = requests.get(Cat_page_url) third_soup = soup(third_r.content,'lxml') names=third_soup.find('div',{'class':'pro_datu'}).find_all('a') for y in names: product_url = y['href'] print(product_url) fourth_r = requests.get(product_url) fourth_soup = soup(fourth_r.content,'lxml') product_names=fourth_soup.find('div',{'class':'prod_nx'}).find('div',{'class':'orat'}).find('span').find_next_sibling('span') Product_number = product_names.text print(Product_number)
https://www.costway.ca/set-of-2-adjustable-pu-leather-backless-bar-stools.html Item No: 78435109 https://www.costway.ca/29-set-of-2-saddle-nailhead-kitchen-counter-chair.html Item No: 13678425 https://www.costway.ca/furniture/chairs.html?p=2 --------------------------------------------------------------------------- AttributeError Traceback (most recent call last) <ipython-input-199-4b0149b6ae67> in <module> 20 fourth_r = requests.get(product_url) 21 fourth_soup = soup(fourth_r.content,'lxml') ---> 22 product_names=fourth_soup.find('div',{'class':'prod_nx'}).find('div',{'class':'orat'}).find('span').find_next_sibling('span') 23 Product_number = product_names.text 24 print(Product_number) AttributeError: 'NoneType' object has no attribute 'find'Thank you very much in advance for looking into it!