Oct-08-2019, 03:46 AM
from bs4 import BeautifulSoup import requests import pandas as pd url = 'https://www.programmableweb.com/apis/directory' api_dict = {} api_no = 0 while True: response = requests.get(url) data = response.text soup = BeautifulSoup(data, 'html.parser') apis = soup.find_all('td',{'class':'views-field views-field-title col-md-3'}) for api in apis: name = api.find('a').text api_no += 1 #print(name) url_tag = soup.find('a',{'title':'Go to next page'}) if url_tag.get('href'): url = 'https://www.programmableweb.com' + url_tag.get('href') #print(url) else: break print('Total APIs: ',api_no) api_dict_df = pd.DataFrame.from_dict(api_dict, orient = 'index', columns = ['API name']) api_dict_df.head() api_dict_df.to_csv('api_detail.csv')