This code will show what's available the group you're interested in (prints out all tr's and td's)
All you have to do is select the elements that you want, and then extract that data.
All you have to do is select the elements that you want, and then extract that data.
from bs4 import BeautifulSoup import requests import pandas as pd import sys def scrape_page(): url = 'https://www.programmableweb.com/apis/directory' api_dict = {} api_no = 0 # while True: response = requests.get(url) if response.status_code == 200: data = response.text else: print(f"Unable to fetch page, bad status: {response.status_code}") sys.exit(-1) soup = BeautifulSoup(data, 'html.parser') tbody = soup.select('.views-table > tbody:nth-child(2)')[0] trs = tbody.find_all('tr') for n, tr in enumerate(trs): tds = tr.find_all('td') for n1, td in enumerate(tds): print(f"\n--------------------- tr_{n}, td_{n1} ---------------------") print(td)