Mar-23-2020, 06:10 PM
I am trying to scrape a website with all the succeeding pages in it. But after the second page, it is repeating itself indefinitely with output from just the second page. What went wrong here? I am not getting any error message. It is continuously scraping just the second page.
from bs4 import BeautifulSoup import requests import pandas as pd url="https://www.programmableweb.com/category/all/apis" while True: response= requests.get(url) response data=response.text soup= BeautifulSoup(data,'html.parser') apis=soup.find_all('tr',{"class":["odd","even"]}) for api in apis: name_tag= api.find('td',{"class":"views-field views-field-pw-version-title"}) name=name_tag.text if name_tag else 'na' des_tag=api.find('td',{'class':'views-field views-field-search-api-excerpt views-field-field-api-description hidden-xs visible-md visible-sm col-md-8'}) des=des_tag.text if des_tag else 'na' category_tag=api.find('td',{'class':'views-field views-field-field-article-primary-category'}) category=category_tag.text if category_tag else 'na' link_tag= api.find('a',{'class':'views-field views-field-pw-version-title'}) link=link_tag.get('href') if link_tag else 'na' print('Name:',name,'\nDescription:', des ,'\nCategory:', category ,'\nLink:', link) url_tag=soup.find('a',{'title':'Go to next page'}) if url_tag.get('href'): url=url+url_tag.get('href') else: break