Jun-05-2018, 12:50 PM
please post enough code to run without work, it forces us to improvise, perhaps causing differnet results:
My attempt:
My attempt:
from bs4 import BeautifulSoup import requests def get_soup(url): soup = None try: response = requests.get(url, headers=headers, timeout=timeout) if response.status_code == 200: html = response.content soup = BeautifulSoup(html, "html.parser") except Exception as exc: print("error::", str(exc)) finally: return soup def get_category_urls(url): soup = get_soup(url) cat_urls = [] try: categories = soup.find('div', attrs={'id': 'menu_oc'}) if categories is not None: for c in categories.findAll('a'): if c['href'] is not None: cat_urls.append(c['href']) except Exception as exc: print("error..", str(exc)) finally: print("category urls::", cat_urls) return cat_urls def main(): url = 'http://www.infantree.net/shop/' soup = get_soup(url) if __name__ == '__main__': main()
Error:error:: name 'headers' is not defined